Skip to content
This repository was archived by the owner on Mar 24, 2022. It is now read-only.

Commit ff5fac8

Browse files
fb55fb55
committed
refactor: Remove Tokenizer statics
1 parent 6f57fec commit ff5fac8

File tree

15 files changed

+302
-331
lines changed

15 files changed

+302
-331
lines changed

‎packages/parse5/lib/common/foreign-content.ts‎

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,3 @@
1-
import { Tokenizer } from '../tokenizer/index.js';
21
import { TAG_NAMES as $, NAMESPACES as NS, ATTRS } from './html.js';
32
import type { TagToken, Attribute } from './token.js';
43

@@ -180,9 +179,7 @@ export function causesExit(startTagToken: TagToken) {
180179
const tn = startTagToken.tagName;
181180
const isFontWithAttrs =
182181
tn === $.FONT &&
183-
(Tokenizer.getTokenAttr(startTagToken, ATTRS.COLOR) !== null ||
184-
Tokenizer.getTokenAttr(startTagToken, ATTRS.SIZE) !== null ||
185-
Tokenizer.getTokenAttr(startTagToken, ATTRS.FACE) !== null);
182+
startTagToken.attrs.some(({ name }) => name === ATTRS.COLOR || name === ATTRS.SIZE || name === ATTRS.FACE);
186183

187184
return isFontWithAttrs || EXITS_FOREIGN_CONTENT.has(tn);
188185
}

‎packages/parse5/lib/common/token.ts‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,16 @@ export interface TagToken extends TokenBase {
6262
location?: LocationWithAttributes;
6363
}
6464

65+
export function getTokenAttr(token: TagToken, attrName: string) {
66+
for (let i = token.attrs.length - 1; i >= 0; i--) {
67+
if (token.attrs[i].name === attrName) {
68+
return token.attrs[i].value;
69+
}
70+
}
71+
72+
return null;
73+
}
74+
6575
export interface CommentToken extends TokenBase {
6676
readonly type: TokenType.COMMENT;
6777
data: string;

‎packages/parse5/lib/extensions/location-info/parser-mixin.ts‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,11 @@
11
import { CommentToken, DoctypeToken, CharacterToken } from '../../common/token';
22
import { Mixin } from '../../utils/mixin.js';
3-
import { Tokenizer } from '../../tokenizer/index.js';
43
import { LocationInfoTokenizerMixin } from './tokenizer-mixin.js';
54
import { TAG_NAMES as $, NAMESPACES as NS } from '../../common/html.js';
65
import type { TreeAdapter, TreeAdapterTypeMap, ElementLocation } from '../../tree-adapters/interface';
76
import type { Parser } from '../../parser/index.js';
87
import type { PositionTrackingPreprocessorMixin } from '../position-tracking/preprocessor-mixin';
9-
import type { Token, TagToken } from '../../common/token.js';
8+
import { TokenType, Token, TagToken } from '../../common/token.js';
109

1110
export class LocationInfoParserMixin<T extends TreeAdapterTypeMap> extends Mixin<Parser<T>> {
1211
treeAdapter: TreeAdapter<T>;
@@ -44,7 +43,7 @@ export class LocationInfoParserMixin<T extends TreeAdapterTypeMap> extends Mixin
4443

4544
// NOTE: For cases like <p> <p> </p> - First 'p' closes without a closing
4645
// tag and for cases like <td> <p> </td> - 'p' closes without a closing tag.
47-
const isClosingEndTag = closingToken.type === Tokenizer.END_TAG_TOKEN && tn === closingToken.tagName;
46+
const isClosingEndTag = closingToken.type === TokenType.END_TAG && tn === closingToken.tagName;
4847
const endLoc: Partial<ElementLocation> = {};
4948
if (isClosingEndTag) {
5049
endLoc.endTag = { ...ctLoc };
@@ -100,7 +99,7 @@ export class LocationInfoParserMixin<T extends TreeAdapterTypeMap> extends Mixin
10099
//NOTE: <body> and <html> are never popped from the stack, so we need to updated
101100
//their end location explicitly.
102101
const requireExplicitUpdate =
103-
token.type === Tokenizer.END_TAG_TOKEN &&
102+
token.type === TokenType.END_TAG &&
104103
(token.tagName === $.HTML || (token.tagName === $.BODY && this.openElements.hasInScope($.BODY)));
105104

106105
if (requireExplicitUpdate) {

‎packages/parse5/lib/extensions/location-info/tokenizer-mixin.test.ts‎

Lines changed: 9 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,14 @@
11
import * as assert from 'assert';
2-
import { Tokenizer } from '../../tokenizer/index.js';
2+
import { Tokenizer, TokenizerMode } from '../../tokenizer/index.js';
33
import { LocationInfoTokenizerMixin } from './tokenizer-mixin.js';
44
import { Mixin } from '../../utils/mixin.js';
5+
import { TokenType } from './../../common/token.js';
56
import { getSubstringByLineCol, normalizeNewLine } from '../../../../../test/utils/common.js';
67

78
it('Location Info (Tokenizer)', () => {
89
const testCases = [
910
{
10-
initialMode: Tokenizer.MODE.DATA,
11+
initialMode: TokenizerMode.DATA,
1112
lastStartTagName: '',
1213
htmlChunks: [
1314
'\r\n',
@@ -59,22 +60,22 @@ it('Location Info (Tokenizer)', () => {
5960
],
6061
},
6162
{
62-
initialMode: Tokenizer.MODE.RCDATA,
63+
initialMode: TokenizerMode.RCDATA,
6364
lastStartTagName: 'title',
6465
htmlChunks: ['<div>Test', ' \n ', 'hey', ' ', 'ya!', '</title>', '<!--Yo-->'],
6566
},
6667
{
67-
initialMode: Tokenizer.MODE.RAWTEXT,
68+
initialMode: TokenizerMode.RAWTEXT,
6869
lastStartTagName: 'style',
6970
htmlChunks: ['.header{', ' \n ', 'color:red;', '\n', '}', '</style>', 'Some', ' ', 'text'],
7071
},
7172
{
72-
initialMode: Tokenizer.MODE.SCRIPT_DATA,
73+
initialMode: TokenizerMode.SCRIPT_DATA,
7374
lastStartTagName: 'script',
7475
htmlChunks: ['var', ' ', 'a=c', ' ', '-', ' ', 'd;', '\n', 'a<--d;', '</script>', '<div>'],
7576
},
7677
{
77-
initialMode: Tokenizer.MODE.PLAINTEXT,
78+
initialMode: TokenizerMode.PLAINTEXT,
7879
lastStartTagName: 'plaintext',
7980
htmlChunks: ['Text', ' \n', 'Test</plaintext><div>'],
8081
},
@@ -97,8 +98,8 @@ it('Location Info (Tokenizer)', () => {
9798
tokenizer.state = testCase.initialMode;
9899
tokenizer.lastStartTagName = testCase.lastStartTagName;
99100

100-
for (let token = tokenizer.getNextToken(), j = 0; token.type !== Tokenizer.EOF_TOKEN; ) {
101-
if (token.type === Tokenizer.HIBERNATION_TOKEN) {
101+
for (let token = tokenizer.getNextToken(), j = 0; token.type !== TokenType.EOF; ) {
102+
if (token.type === TokenType.HIBERNATION) {
102103
continue;
103104
}
104105

‎packages/parse5/lib/extensions/location-info/tokenizer-mixin.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
import { Mixin } from '../../utils/mixin.js';
22
import { Tokenizer } from '../../tokenizer/index.js';
33
import { PositionTrackingPreprocessorMixin } from '../position-tracking/preprocessor-mixin.js';
4-
import { Location, LocationWithAttributes } from '../../common/token.js';
4+
import { TokenType, Location, LocationWithAttributes } from '../../common/token.js';
55

66
export class LocationInfoTokenizerMixin extends Mixin<Tokenizer> {
77
posTracker: PositionTrackingPreprocessorMixin;
@@ -97,7 +97,7 @@ export class LocationInfoTokenizerMixin extends Mixin<Tokenizer> {
9797
currentCharacterToken.location!.endOffset = ctLoc.startOffset;
9898
}
9999

100-
if (this.currentToken!.type === Tokenizer.EOF_TOKEN) {
100+
if (this.currentToken!.type === TokenType.EOF) {
101101
ctLoc.endLine = ctLoc.startLine;
102102
ctLoc.endCol = ctLoc.startCol;
103103
ctLoc.endOffset = ctLoc.startOffset;

‎packages/parse5/lib/parser/index.test.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ import { NAMESPACES as NS } from '../common/html.js';
77

88
const origParseFragment = Parser.prototype.parseFragment;
99

10-
generateParsingTests('parser', 'Parser', { skipFragments: false }, (test, opts) => ({
10+
generateParsingTests('parser', 'Parser', {}, (test, opts) => ({
1111
node: test.fragmentContext
1212
? parse5.parseFragment(test.fragmentContext, test.input, opts)
1313
: parse5.parse(test.input, opts),

0 commit comments

Comments
 (0)