You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
239 lines
6.0 KiB
239 lines
6.0 KiB
export var tokenize = initializeDocument |
|
|
|
import codes from '../character/codes.mjs' |
|
import markdownLineEnding from '../character/markdown-line-ending.mjs' |
|
import constants from '../constant/constants.mjs' |
|
import types from '../constant/types.mjs' |
|
import spaceFactory from '../tokenize/factory-space.mjs' |
|
import blank from '../tokenize/partial-blank-line.mjs' |
|
|
|
var containerConstruct = {tokenize: tokenizeContainer} |
|
var lazyFlowConstruct = {tokenize: tokenizeLazyFlow} |
|
|
|
function initializeDocument(effects) { |
|
var self = this |
|
var stack = [] |
|
var continued = 0 |
|
var inspectConstruct = {tokenize: tokenizeInspect, partial: true} |
|
var inspectResult |
|
var childFlow |
|
var childToken |
|
|
|
return start |
|
|
|
function start(code) { |
|
if (continued < stack.length) { |
|
self.containerState = stack[continued][1] |
|
return effects.attempt( |
|
stack[continued][0].continuation, |
|
documentContinue, |
|
documentContinued |
|
)(code) |
|
} |
|
|
|
return documentContinued(code) |
|
} |
|
|
|
function documentContinue(code) { |
|
continued++ |
|
return start(code) |
|
} |
|
|
|
function documentContinued(code) { |
|
// If we’re in a concrete construct (such as when expecting another line of |
|
// HTML, or we resulted in lazy content), we can immediately start flow. |
|
if (inspectResult && inspectResult.flowContinue) { |
|
return flowStart(code) |
|
} |
|
|
|
self.interrupt = |
|
childFlow && |
|
childFlow.currentConstruct && |
|
childFlow.currentConstruct.interruptible |
|
self.containerState = {} |
|
return effects.attempt( |
|
containerConstruct, |
|
containerContinue, |
|
flowStart |
|
)(code) |
|
} |
|
|
|
function containerContinue(code) { |
|
stack.push([self.currentConstruct, self.containerState]) |
|
self.containerState = undefined |
|
return documentContinued(code) |
|
} |
|
|
|
function flowStart(code) { |
|
if (code === codes.eof) { |
|
exitContainers(0, true) |
|
effects.consume(code) |
|
return |
|
} |
|
|
|
childFlow = childFlow || self.parser.flow(self.now()) |
|
|
|
effects.enter(types.chunkFlow, { |
|
contentType: constants.contentTypeFlow, |
|
previous: childToken, |
|
_tokenizer: childFlow |
|
}) |
|
|
|
return flowContinue(code) |
|
} |
|
|
|
function flowContinue(code) { |
|
if (code === codes.eof) { |
|
continueFlow(effects.exit(types.chunkFlow)) |
|
return flowStart(code) |
|
} |
|
|
|
if (markdownLineEnding(code)) { |
|
effects.consume(code) |
|
continueFlow(effects.exit(types.chunkFlow)) |
|
return effects.check(inspectConstruct, documentAfterPeek) |
|
} |
|
|
|
effects.consume(code) |
|
return flowContinue |
|
} |
|
|
|
function documentAfterPeek(code) { |
|
exitContainers( |
|
inspectResult.continued, |
|
inspectResult && inspectResult.flowEnd |
|
) |
|
continued = 0 |
|
return start(code) |
|
} |
|
|
|
function continueFlow(token) { |
|
if (childToken) childToken.next = token |
|
childToken = token |
|
childFlow.lazy = inspectResult && inspectResult.lazy |
|
childFlow.defineSkip(token.start) |
|
childFlow.write(self.sliceStream(token)) |
|
} |
|
|
|
function exitContainers(size, end) { |
|
var index = stack.length |
|
|
|
// Close the flow. |
|
if (childFlow && end) { |
|
childFlow.write([codes.eof]) |
|
childToken = childFlow = undefined |
|
} |
|
|
|
// Exit open containers. |
|
while (index-- > size) { |
|
self.containerState = stack[index][1] |
|
stack[index][0].exit.call(self, effects) |
|
} |
|
|
|
stack.length = size |
|
} |
|
|
|
function tokenizeInspect(effects, ok) { |
|
var subcontinued = 0 |
|
|
|
inspectResult = {} |
|
|
|
return inspectStart |
|
|
|
function inspectStart(code) { |
|
if (subcontinued < stack.length) { |
|
self.containerState = stack[subcontinued][1] |
|
return effects.attempt( |
|
stack[subcontinued][0].continuation, |
|
inspectContinue, |
|
inspectLess |
|
)(code) |
|
} |
|
|
|
// If we’re continued but in a concrete flow, we can’t have more |
|
// containers. |
|
if (childFlow.currentConstruct && childFlow.currentConstruct.concrete) { |
|
inspectResult.flowContinue = true |
|
return inspectDone(code) |
|
} |
|
|
|
self.interrupt = |
|
childFlow.currentConstruct && childFlow.currentConstruct.interruptible |
|
self.containerState = {} |
|
return effects.attempt( |
|
containerConstruct, |
|
inspectFlowEnd, |
|
inspectDone |
|
)(code) |
|
} |
|
|
|
function inspectContinue(code) { |
|
subcontinued++ |
|
return self.containerState._closeFlow |
|
? inspectFlowEnd(code) |
|
: inspectStart(code) |
|
} |
|
|
|
function inspectLess(code) { |
|
if (childFlow.currentConstruct && childFlow.currentConstruct.lazy) { |
|
// Maybe another container? |
|
self.containerState = {} |
|
return effects.attempt( |
|
containerConstruct, |
|
inspectFlowEnd, |
|
// Maybe flow, or a blank line? |
|
effects.attempt( |
|
lazyFlowConstruct, |
|
inspectFlowEnd, |
|
effects.check(blank, inspectFlowEnd, inspectLazy) |
|
) |
|
)(code) |
|
} |
|
|
|
// Otherwise we’re interrupting. |
|
return inspectFlowEnd(code) |
|
} |
|
|
|
function inspectLazy(code) { |
|
// Act as if all containers are continued. |
|
subcontinued = stack.length |
|
inspectResult.lazy = true |
|
inspectResult.flowContinue = true |
|
return inspectDone(code) |
|
} |
|
|
|
// We’re done with flow if we have more containers, or an interruption. |
|
function inspectFlowEnd(code) { |
|
inspectResult.flowEnd = true |
|
return inspectDone(code) |
|
} |
|
|
|
function inspectDone(code) { |
|
inspectResult.continued = subcontinued |
|
self.interrupt = self.containerState = undefined |
|
return ok(code) |
|
} |
|
} |
|
} |
|
|
|
function tokenizeContainer(effects, ok, nok) { |
|
return spaceFactory( |
|
effects, |
|
effects.attempt(this.parser.constructs.document, ok, nok), |
|
types.linePrefix, |
|
this.parser.constructs.disable.null.indexOf('codeIndented') > -1 |
|
? undefined |
|
: constants.tabSize |
|
) |
|
} |
|
|
|
function tokenizeLazyFlow(effects, ok, nok) { |
|
return spaceFactory( |
|
effects, |
|
effects.lazy(this.parser.constructs.flow, ok, nok), |
|
types.linePrefix, |
|
this.parser.constructs.disable.null.indexOf('codeIndented') > -1 |
|
? undefined |
|
: constants.tabSize |
|
) |
|
}
|
|
|