Skip to content

Commit 20f065d

Browse files
JackWang032JackWang032
authored
fix: update isContainCaret judgment when caret position token is whit… (#390)
* fix: update isContainCaret judgment when caret position token is whitespace * fix: remove unnecessary +Infinity
1 parent 05134bc commit 20f065d

23 files changed

+220
-32
lines changed

‎src/parser/common/basicSQL.ts

+2-1
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,7 @@ export abstract class BasicSQL<
8383
*/
8484
protected abstract createEntityCollector(
8585
input: string,
86+
allTokens?: Token[],
8687
caretTokenIndex?: number
8788
): EntityCollector;
8889

@@ -378,7 +379,7 @@ export abstract class BasicSQL<
378379
? findCaretTokenIndex(caretPosition, allTokens)
379380
: void 0;
380381

381-
const collectListener = this.createEntityCollector(input, caretTokenIndex);
382+
const collectListener = this.createEntityCollector(input, allTokens, caretTokenIndex);
382383
// const parser = this.createParserWithCache(input);
383384

384385
// parser.entityCollecting = true;

‎src/parser/common/entityCollector.ts

+22-3
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
import { ParserRuleContext } from 'antlr4ng';
1+
import { ParserRuleContext, Token } from 'antlr4ng';
22
import { EntityContextType } from './types';
33
import { WordPosition, TextPosition } from './textAndWord';
44
import { ctxToText, ctxToWord } from './textAndWord';
@@ -96,15 +96,17 @@ export function toEntityContext(
9696
* @todo: [may be need] Combine the entities in each clause.
9797
*/
9898
export abstract class EntityCollector {
99-
constructor(input: string, caretTokenIndex?: number) {
99+
constructor(input: string, allTokens?: Token[], caretTokenIndex?: number) {
100100
this._input = input;
101+
this._allTokens = allTokens || [];
101102
this._caretTokenIndex = caretTokenIndex ?? -1;
102103
this._entitiesSet = new Set();
103104
this._stmtStack = new SimpleStack();
104105
this._entityStack = new SimpleStack();
105106
this._rootStmt = null;
106107
}
107108
private readonly _input: string;
109+
private readonly _allTokens: Token[];
108110
private readonly _caretTokenIndex: number;
109111
private readonly _entitiesSet: Set<EntityContext>;
110112
/** Staging statements that have already entered. */
@@ -136,14 +138,31 @@ export abstract class EntityCollector {
136138
this._rootStmt = null;
137139
}
138140

141+
/**
142+
* The antlr4 will ignore hidden tokens, if we type whitespace at the end of a statement,
143+
* the whitespace token will not as stop token, so we consider the whitespace token as a part of the nonhidden token in front of it
144+
*/
145+
protected getPrevNonHiddenTokenIndex(caretTokenIndex: number) {
146+
if (this._allTokens[caretTokenIndex].channel !== Token.HIDDEN_CHANNEL)
147+
return caretTokenIndex;
148+
for (let i = caretTokenIndex - 1; i >= 0; i--) {
149+
const token = this._allTokens[i];
150+
if (token.channel !== Token.HIDDEN_CHANNEL) {
151+
// If prev nonhidden token is ';', the current token does not belong to any statement.
152+
return token.text === ';' ? Infinity : token.tokenIndex;
153+
}
154+
}
155+
return Infinity;
156+
}
157+
139158
protected pushStmt(ctx: ParserRuleContext, type: StmtContextType) {
140159
let isContainCaret: boolean | undefined;
141160
if (this._caretTokenIndex >= 0) {
142161
isContainCaret =
143162
!!ctx.start &&
144163
!!ctx.stop &&
145164
ctx.start.tokenIndex <= this._caretTokenIndex &&
146-
ctx.stop.tokenIndex >= this._caretTokenIndex;
165+
ctx.stop.tokenIndex >= this.getPrevNonHiddenTokenIndex(this._caretTokenIndex);
147166
}
148167
const stmtContext = toStmtContext(
149168
ctx,

‎src/parser/flink/index.ts

+2-2
Original file line numberDiff line numberDiff line change
@@ -37,8 +37,8 @@ export class FlinkSQL extends BasicSQL<FlinkSqlLexer, ProgramContext, FlinkSqlPa
3737
return new FlinkSqlSplitListener();
3838
}
3939

40-
protected createEntityCollector(input: string, caretTokenIndex?: number) {
41-
return new FlinkEntityCollector(input, caretTokenIndex);
40+
protected createEntityCollector(input: string, allTokens?: Token[], caretTokenIndex?: number) {
41+
return new FlinkEntityCollector(input, allTokens, caretTokenIndex);
4242
}
4343

4444
protected processCandidates(

‎src/parser/hive/index.ts

+2-2
Original file line numberDiff line numberDiff line change
@@ -38,8 +38,8 @@ export class HiveSQL extends BasicSQL<HiveSqlLexer, ProgramContext, HiveSqlParse
3838
return new HiveSqlSplitListener();
3939
}
4040

41-
protected createEntityCollector(input: string, caretTokenIndex?: number) {
42-
return new HiveEntityCollector(input, caretTokenIndex);
41+
protected createEntityCollector(input: string, allTokens?: Token[], caretTokenIndex?: number) {
42+
return new HiveEntityCollector(input, allTokens, caretTokenIndex);
4343
}
4444

4545
protected processCandidates(

‎src/parser/impala/index.ts

+2-2
Original file line numberDiff line numberDiff line change
@@ -36,8 +36,8 @@ export class ImpalaSQL extends BasicSQL<ImpalaSqlLexer, ProgramContext, ImpalaSq
3636
return new ImpalaSqlSplitListener();
3737
}
3838

39-
protected createEntityCollector(input: string, caretTokenIndex?: number) {
40-
return new ImpalaEntityCollector(input, caretTokenIndex);
39+
protected createEntityCollector(input: string, allTokens?: Token[], caretTokenIndex?: number) {
40+
return new ImpalaEntityCollector(input, allTokens, caretTokenIndex);
4141
}
4242

4343
protected processCandidates(

‎src/parser/mysql/index.ts

+2-2
Original file line numberDiff line numberDiff line change
@@ -36,8 +36,8 @@ export class MySQL extends BasicSQL<MySqlLexer, ProgramContext, MySqlParser> {
3636
return new MysqlSplitListener();
3737
}
3838

39-
protected createEntityCollector(input: string, caretTokenIndex?: number) {
40-
return new MySqlEntityCollector(input, caretTokenIndex);
39+
protected createEntityCollector(input: string, allTokens?: Token[], caretTokenIndex?: number) {
40+
return new MySqlEntityCollector(input, allTokens, caretTokenIndex);
4141
}
4242

4343
protected processCandidates(

‎src/parser/postgresql/index.ts

+2-2
Original file line numberDiff line numberDiff line change
@@ -41,8 +41,8 @@ export class PostgreSQL extends BasicSQL<PostgreSqlLexer, ProgramContext, Postgr
4141
return new PostgreSqlSplitListener();
4242
}
4343

44-
protected createEntityCollector(input: string, caretTokenIndex?: number) {
45-
return new PostgreSqlEntityCollector(input, caretTokenIndex);
44+
protected createEntityCollector(input: string, allTokens?: Token[], caretTokenIndex?: number) {
45+
return new PostgreSqlEntityCollector(input, allTokens, caretTokenIndex);
4646
}
4747

4848
protected processCandidates(

‎src/parser/spark/index.ts

+2-2
Original file line numberDiff line numberDiff line change
@@ -36,8 +36,8 @@ export class SparkSQL extends BasicSQL<SparkSqlLexer, ProgramContext, SparkSqlPa
3636
return new SparkSqlSplitListener();
3737
}
3838

39-
protected createEntityCollector(input: string, caretTokenIndex?: number) {
40-
return new SparkEntityCollector(input, caretTokenIndex);
39+
protected createEntityCollector(input: string, allTokens?: Token[], caretTokenIndex?: number) {
40+
return new SparkEntityCollector(input, allTokens, caretTokenIndex);
4141
}
4242

4343
protected processCandidates(

‎src/parser/trino/index.ts

+2-2
Original file line numberDiff line numberDiff line change
@@ -23,8 +23,8 @@ export class TrinoSQL extends BasicSQL<TrinoSqlLexer, ProgramContext, TrinoSqlPa
2323
return new TrinoSqlSplitListener();
2424
}
2525

26-
protected createEntityCollector(input: string, caretTokenIndex?: number) {
27-
return new TrinoEntityCollector(input, caretTokenIndex);
26+
protected createEntityCollector(input: string, allTokens?: Token[], caretTokenIndex?: number) {
27+
return new TrinoEntityCollector(input, allTokens, caretTokenIndex);
2828
}
2929

3030
protected preferredRules: Set<number> = new Set([

‎test/parser/flink/suggestion/fixtures/suggestionWithEntity.sql

+5-1
Original file line numberDiff line numberDiff line change
@@ -8,4 +8,8 @@ INSERT INTO insert_tb PARTITION (country, state) SELECT col1, col2, country, sta
88

99
CREATE TABLE IF NOT EXISTS derived_table WITH ('connector' = 'kafka') AS SELECT FROM origin_table;
1010

11-
CREATE TABLE IF NOT EXISTS derived_table WITH ('connector' = 'kafka') AS SELECT id, FROM origin_table;
11+
CREATE TABLE IF NOT EXISTS derived_table WITH ('connector' = 'kafka') AS SELECT id, FROM origin_table;
12+
13+
SELECT id FROM tb WHERE
14+
15+
SELECT id FROM tb GROUP BY ;

‎test/parser/flink/suggestion/suggestionWithEntity.test.ts

+20
Original file line numberDiff line numberDiff line change
@@ -157,4 +157,24 @@ describe('Flink SQL Syntax Suggestion with collect entity', () => {
157157
expect(entities[1].entityContextType).toBe(EntityContextType.TABLE);
158158
expect(entities[1].belongStmt.isContainCaret).toBeTruthy();
159159
});
160+
161+
test('isContainCaret should be truthy if caret position is whitespace at the end of statement', () => {
162+
const pos: CaretPosition = {
163+
lineNumber: 13,
164+
column: 25,
165+
};
166+
const sql = commentOtherLine(syntaxSql, pos.lineNumber);
167+
const entities = flink.getAllEntities(sql, pos);
168+
expect(entities[0].belongStmt.isContainCaret).toBeTruthy();
169+
});
170+
171+
test('isContainCaret should be falsy if caret position is whitespace after semicolon', () => {
172+
const pos: CaretPosition = {
173+
lineNumber: 15,
174+
column: 32,
175+
};
176+
const sql = commentOtherLine(syntaxSql, pos.lineNumber);
177+
const entities = flink.getAllEntities(sql, pos);
178+
expect(entities[0].belongStmt.isContainCaret).toBeFalsy();
179+
});
160180
});

‎test/parser/hive/suggestion/fixtures/suggestionWithEntity.sql

+5-1
Original file line numberDiff line numberDiff line change
@@ -20,4 +20,8 @@ INSERT INTO insert_tb PARTITION (country, state) SELECT col1, col2, country, sta
2020

2121
CREATE TABLE IF NOT EXISTS derived_table AS SELECT FROM origin_table
2222

23-
CREATE TABLE IF NOT EXISTS derived_table AS SELECT id, FROM origin_table
23+
CREATE TABLE IF NOT EXISTS derived_table AS SELECT id, FROM origin_table
24+
25+
SELECT id FROM tb WHERE
26+
27+
SELECT id FROM tb GROUP BY ;

‎test/parser/hive/suggestion/suggestionWithEntity.test.ts

+28-8
Original file line numberDiff line numberDiff line change
@@ -118,12 +118,12 @@ describe('Hive SQL Syntax Suggestion with collect entity', () => {
118118
expect(entities.length).toBe(2);
119119
expect(entities[0].text).toBe('a');
120120
expect(entities[0].entityContextType).toBe(EntityContextType.TABLE);
121-
expect(entities[0].belongStmt.isContainCaret).toBeFalsy();
121+
expect(entities[0].belongStmt.isContainCaret).toBeTruthy();
122122
expect(entities[0].belongStmt.rootStmt.isContainCaret).toBeTruthy();
123123

124124
expect(entities[1].text).toBe('b');
125125
expect(entities[1].entityContextType).toBe(EntityContextType.TABLE);
126-
expect(entities[1].belongStmt.isContainCaret).toBeFalsy();
126+
expect(entities[1].belongStmt.isContainCaret).toBeTruthy();
127127
expect(entities[1].belongStmt.rootStmt.isContainCaret).toBeTruthy();
128128
});
129129

@@ -145,12 +145,12 @@ describe('Hive SQL Syntax Suggestion with collect entity', () => {
145145
expect(entities.length).toBe(2);
146146
expect(entities[0].text).toBe('a');
147147
expect(entities[0].entityContextType).toBe(EntityContextType.TABLE);
148-
expect(entities[0].belongStmt.isContainCaret).toBeFalsy();
148+
expect(entities[0].belongStmt.isContainCaret).toBeTruthy();
149149
expect(entities[0].belongStmt.rootStmt.isContainCaret).toBeTruthy();
150150

151151
expect(entities[1].text).toBe('b');
152152
expect(entities[1].entityContextType).toBe(EntityContextType.TABLE);
153-
expect(entities[1].belongStmt.isContainCaret).toBeFalsy();
153+
expect(entities[1].belongStmt.isContainCaret).toBeTruthy();
154154
expect(entities[1].belongStmt.rootStmt.isContainCaret).toBeTruthy();
155155
});
156156

@@ -172,12 +172,12 @@ describe('Hive SQL Syntax Suggestion with collect entity', () => {
172172
expect(entities.length).toBe(2);
173173
expect(entities[0].text).toBe('page_view_stg');
174174
expect(entities[0].entityContextType).toBe(EntityContextType.TABLE);
175-
expect(entities[0].belongStmt.isContainCaret).toBeFalsy();
175+
expect(entities[0].belongStmt.isContainCaret).toBeTruthy();
176176
expect(entities[0].belongStmt.rootStmt.isContainCaret).toBeTruthy();
177177

178178
expect(entities[1].text).toBe('page_view');
179179
expect(entities[1].entityContextType).toBe(EntityContextType.TABLE);
180-
expect(entities[1].belongStmt.isContainCaret).toBeFalsy();
180+
expect(entities[1].belongStmt.isContainCaret).toBeTruthy();
181181
expect(entities[1].belongStmt.rootStmt.isContainCaret).toBeTruthy();
182182
});
183183

@@ -199,12 +199,12 @@ describe('Hive SQL Syntax Suggestion with collect entity', () => {
199199
expect(entities.length).toBe(2);
200200
expect(entities[0].text).toBe('page_view_stg');
201201
expect(entities[0].entityContextType).toBe(EntityContextType.TABLE);
202-
expect(entities[0].belongStmt.isContainCaret).toBeFalsy();
202+
expect(entities[0].belongStmt.isContainCaret).toBeTruthy();
203203
expect(entities[0].belongStmt.rootStmt.isContainCaret).toBeTruthy();
204204

205205
expect(entities[1].text).toBe('page_view');
206206
expect(entities[1].entityContextType).toBe(EntityContextType.TABLE);
207-
expect(entities[1].belongStmt.isContainCaret).toBeFalsy();
207+
expect(entities[1].belongStmt.isContainCaret).toBeTruthy();
208208
expect(entities[1].belongStmt.rootStmt.isContainCaret).toBeTruthy();
209209
});
210210

@@ -307,4 +307,24 @@ describe('Hive SQL Syntax Suggestion with collect entity', () => {
307307
expect(entities[1].entityContextType).toBe(EntityContextType.TABLE);
308308
expect(entities[1].belongStmt.isContainCaret).toBeTruthy();
309309
});
310+
311+
test('isContainCaret should be truthy if caret position is whitespace at the end of statement', () => {
312+
const pos: CaretPosition = {
313+
lineNumber: 25,
314+
column: 25,
315+
};
316+
const sql = commentOtherLine(syntaxSql, pos.lineNumber);
317+
const entities = hive.getAllEntities(sql, pos);
318+
expect(entities[0].belongStmt.isContainCaret).toBeTruthy();
319+
});
320+
321+
test('isContainCaret should be falsy if caret position is whitespace after semicolon', () => {
322+
const pos: CaretPosition = {
323+
lineNumber: 27,
324+
column: 32,
325+
};
326+
const sql = commentOtherLine(syntaxSql, pos.lineNumber);
327+
const entities = hive.getAllEntities(sql, pos);
328+
expect(entities[0].belongStmt.isContainCaret).toBeFalsy();
329+
});
310330
});

‎test/parser/impala/suggestion/fixtures/suggestionWithEntity.sql

+4
Original file line numberDiff line numberDiff line change
@@ -9,3 +9,7 @@ INSERT INTO insert_tb SELECT id, FROM from_tb;
99
CREATE TABLE sorted_census_data AS SELECT FROM unsorted_census_data;
1010

1111
CREATE TABLE sorted_census_data AS SELECT id, FROM unsorted_census_data;
12+
13+
SELECT id FROM tb WHERE
14+
15+
SELECT id FROM tb GROUP BY ;

‎test/parser/impala/suggestion/suggestionWithEntity.test.ts

+20
Original file line numberDiff line numberDiff line change
@@ -155,4 +155,24 @@ describe('Impala SQL Syntax Suggestion with collect entity', () => {
155155
expect(entities[1].entityContextType).toBe(EntityContextType.TABLE);
156156
expect(entities[1].belongStmt.isContainCaret).toBeTruthy();
157157
});
158+
159+
test('isContainCaret should be truthy if caret position is whitespace at the end of statement', () => {
160+
const pos: CaretPosition = {
161+
lineNumber: 13,
162+
column: 25,
163+
};
164+
const sql = commentOtherLine(syntaxSql, pos.lineNumber);
165+
const entities = impala.getAllEntities(sql, pos);
166+
expect(entities[0].belongStmt.isContainCaret).toBeTruthy();
167+
});
168+
169+
test('isContainCaret should be falsy if caret position is whitespace after semicolon', () => {
170+
const pos: CaretPosition = {
171+
lineNumber: 15,
172+
column: 32,
173+
};
174+
const sql = commentOtherLine(syntaxSql, pos.lineNumber);
175+
const entities = impala.getAllEntities(sql, pos);
176+
expect(entities[0].belongStmt.isContainCaret).toBeFalsy();
177+
});
158178
});

‎test/parser/mysql/suggestion/fixtures/suggestionWithEntity.sql

+5-1
Original file line numberDiff line numberDiff line change
@@ -8,4 +8,8 @@ INSERT INTO insert_tb SELECT id, age, FROM from_tb;
88

99
CREATE TABLE sorted_census_data AS SELECT FROM unsorted_census_data;
1010

11-
CREATE TABLE sorted_census_data AS SELECT id, age, FROM unsorted_census_data;
11+
CREATE TABLE sorted_census_data AS SELECT id, age, FROM unsorted_census_data;
12+
13+
SELECT id FROM tb WHERE
14+
15+
SELECT id FROM tb GROUP BY ;

‎test/parser/mysql/suggestion/suggestionWithEntity.test.ts

+20
Original file line numberDiff line numberDiff line change
@@ -153,4 +153,24 @@ describe('MySQL Syntax Suggestion with collect entity', () => {
153153
expect(entities[1].entityContextType).toBe(EntityContextType.TABLE);
154154
expect(entities[1].belongStmt.isContainCaret).toBeTruthy();
155155
});
156+
157+
test('isContainCaret should be truthy if caret position is whitespace at the end of statement', () => {
158+
const pos: CaretPosition = {
159+
lineNumber: 13,
160+
column: 25,
161+
};
162+
const sql = commentOtherLine(syntaxSql, pos.lineNumber);
163+
const entities = mysql.getAllEntities(sql, pos);
164+
expect(entities[0].belongStmt.isContainCaret).toBeTruthy();
165+
});
166+
167+
test('isContainCaret should be falsy if caret position is whitespace after semicolon', () => {
168+
const pos: CaretPosition = {
169+
lineNumber: 15,
170+
column: 32,
171+
};
172+
const sql = commentOtherLine(syntaxSql, pos.lineNumber);
173+
const entities = mysql.getAllEntities(sql, pos);
174+
expect(entities[0].belongStmt.isContainCaret).toBeFalsy();
175+
});
156176
});

‎test/parser/postgresql/suggestion/fixtures/suggestionWithEntity.sql

+5-1
Original file line numberDiff line numberDiff line change
@@ -10,4 +10,8 @@ CREATE TABLE sorted_census_data AS SELECT FROM unsorted_census_data;
1010

1111
CREATE TABLE sorted_census_data AS SELECT id, age, FROM unsorted_census_data;
1212

13-
ALTER TABLE my_table DROP a_column;
13+
ALTER TABLE my_table DROP a_column;
14+
15+
SELECT id FROM tb WHERE
16+
17+
SELECT id FROM tb GROUP BY ;

‎test/parser/postgresql/suggestion/suggestionWithEntity.test.ts

+20
Original file line numberDiff line numberDiff line change
@@ -174,4 +174,24 @@ describe('PostgreSql Syntax Suggestion with collect entity', () => {
174174
expect(entities[0].entityContextType).toBe(EntityContextType.TABLE);
175175
expect(entities[0].belongStmt?.isContainCaret).toBeTruthy();
176176
});
177+
178+
test('isContainCaret should be truthy if caret position is whitespace at the end of statement', () => {
179+
const pos: CaretPosition = {
180+
lineNumber: 15,
181+
column: 25,
182+
};
183+
const sql = commentOtherLine(syntaxSql, pos.lineNumber);
184+
const entities = postgre.getAllEntities(sql, pos);
185+
expect(entities[0].belongStmt.isContainCaret).toBeTruthy();
186+
});
187+
188+
test('isContainCaret should be falsy if caret position is whitespace after semicolon', () => {
189+
const pos: CaretPosition = {
190+
lineNumber: 17,
191+
column: 32,
192+
};
193+
const sql = commentOtherLine(syntaxSql, pos.lineNumber);
194+
const entities = postgre.getAllEntities(sql, pos);
195+
expect(entities[0].belongStmt.isContainCaret).toBeFalsy();
196+
});
177197
});

0 commit comments

Comments
 (0)