Skip to content

Commit 6d22737

Browse files
m-mohrm-mohr
committed
Fix #8, support circular schemas, remove support for beta versions, other improvements, v1.1.0
1 parent b375b76 commit 6d22737

File tree

4 files changed

+114
-141
lines changed

4 files changed

+114
-141
lines changed

‎COMPARISON.md

+1-1
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ Here I'd like to give an overview of what the validators are capable of and what
1414

1515
| | Python Validator | PySTAC | STAC Node Validator |
1616
| :------------------------- | ------------------------------------------ | ------------------- | ------------------- |
17-
| Validator Version | 1.0.1 | 0.5.2 | 1.0.1 |
17+
| Validator Version | 1.0.1 | 0.5.2 | 1.1.0 |
1818
| Language | Python 3.6 | Python 3 | NodeJS |
1919
| CLI | Yes | No | Yes |
2020
| Programmatic | Yes | Yes | Planned |

‎README.md

+3-2
Original file line numberDiff line numberDiff line change
@@ -6,11 +6,12 @@ See the [STAC Validator Comparison](COMPARISON.md) for the features supported by
66

77
## Versions
88

9-
**Current version: 1.0.1**
9+
**Current version: 1.1.0**
1010

1111
| STAC Node Validator Version | Supported STAC Versions |
1212
| --------------------------- | ----------------------- |
13-
| 0.4.x / 1.0.x | >= 1.0.0-beta.2 |
13+
| 1.1.x | >= 1.0.0-rc.1 |
14+
| 0.4.x / 1.0.x | >= 1.0.0-beta.2 and < 1.0.0-rc.3 |
1415
| 0.3.0 | 1.0.0-beta.2 |
1516
| 0.2.1 | 1.0.0-beta.1 |
1617

‎index.js

+108-136
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
1-
const $RefParser = require("@apidevtools/json-schema-ref-parser");
21
const Ajv = require('ajv');
2+
const axios = require('axios');
33
const formats = require('ajv-formats-draft2019/formats');
44
const iriFormats = require('./iri.js');
55
const fs = require('fs-extra');
@@ -11,35 +11,15 @@ const {diffStringsUnified} = require('jest-diff');
1111
const package = require('./package.json');
1212

1313
let DEBUG = false;
14-
let COMPILED = {};
15-
let SHORTCUTS = [
16-
'checksum', // legacy
17-
'collection-assets', // now in core
18-
'datacube', // now in stac-extensions org
19-
'eo',
20-
'item-assets', // now in stac-extensions org
21-
'label', // now in stac-extensions org
22-
'pointcloud', // now in stac-extensions org
23-
'processing', // now in stac-extensions org
24-
'projection',
25-
'sar', // now in stac-extensions org
26-
'sat', // now in stac-extensions org
27-
'scientific',
28-
'single-file-stac', // now in stac-extensions org
29-
'tiled-assets', // now in stac-extensions org
30-
'timestamps', // now in stac-extensions org
31-
'version', // now in stac-extensions org
32-
'view'
33-
];
3414
let ajv = new Ajv({
3515
formats: Object.assign(formats, iriFormats),
3616
allErrors: true,
37-
missingRefs: "ignore",
38-
addUsedSchema: false,
39-
logger: DEBUG ? console : false
17+
logger: DEBUG ? console : false,
18+
loadSchema: loadJsonFromUri
4019
});
4120
let verbose = false;
4221
let schemaMap = {};
22+
let schemaFolder = null;
4323

4424
async function run() {
4525
console.log(`STAC Node Validator v${package.version}\n`);
@@ -64,14 +44,13 @@ async function run() {
6444
process.env["NODE_TLS_REJECT_UNAUTHORIZED"] = 0;
6545
}
6646

67-
let schemaFolder = null;
6847
if (typeof args.schemas === 'string') {
6948
let stat = await fs.lstat(args.schemas);
7049
if (stat.isDirectory()) {
71-
schemaFolder = args.schemas;
50+
schemaFolder = normalizePath(args.schemas);
7251
}
7352
else {
74-
throw new Error('Schema folder is not a valid directory');
53+
throw new Error('Schema folder is not a valid STAC directory');
7554
}
7655
}
7756

@@ -109,38 +88,36 @@ async function run() {
10988
let json;
11089
console.log(`- ${file}`);
11190
try {
112-
if (isUrl(file)) {
113-
// For simplicity, we just load the URLs with $RefParser, so we don't need another dependency.
114-
json = await $RefParser.parse(file);
115-
if (doLint) {
116-
console.warn("-- Linting not supported for remote files");
117-
}
118-
if (doFormat) {
119-
console.warn("-- Formatting not supported for remote files");
120-
}
121-
}
122-
else {
91+
let fileIsUrl = isUrl(file);
92+
if (!fileIsUrl && (doLint || doFormat)) {
12393
let fileContent = await fs.readFile(file, "utf8");
12494
json = JSON.parse(fileContent);
125-
if (doLint || doFormat) {
126-
const expectedContent = JSON.stringify(json, null, 2);
127-
if (!matchFile(fileContent, expectedContent)) {
128-
stats.malformed++;
129-
if (doLint) {
130-
console.warn("-- Lint: File is malformed -> use `--format` to fix the issue");
131-
if (verbose) {
132-
console.log(diffStringsUnified(fileContent, expectedContent));
133-
}
134-
}
135-
if (doFormat) {
136-
console.warn("-- Format: File was malformed -> fixed the issue");
137-
await fs.writeFile(file, expectedContent);
95+
const expectedContent = JSON.stringify(json, null, 2);
96+
if (!matchFile(fileContent, expectedContent)) {
97+
stats.malformed++;
98+
if (doLint) {
99+
console.warn("-- Lint: File is malformed -> use `--format` to fix the issue");
100+
if (verbose) {
101+
console.log(diffStringsUnified(fileContent, expectedContent));
138102
}
139103
}
140-
else if (doLint && verbose) {
141-
console.warn("-- Lint: File is well-formed");
104+
if (doFormat) {
105+
console.warn("-- Format: File was malformed -> fixed the issue");
106+
await fs.writeFile(file, expectedContent);
142107
}
143108
}
109+
else if (doLint && verbose) {
110+
console.warn("-- Lint: File is well-formed");
111+
}
112+
}
113+
else {
114+
json = await loadJsonFromUri(file);
115+
if (fileIsUrl && (doLint || doFormat)) {
116+
let what = [];
117+
doLint && what.push('Linting');
118+
doLint && what.push('Formatting');
119+
console.warn(`-- ${what.join(' and ')} not supported for remote files`);
120+
}
144121
}
145122
}
146123
catch(error) {
@@ -187,53 +164,67 @@ async function run() {
187164
fileValid = false;
188165
continue;
189166
}
190-
else if (versions.compare(data.stac_version, '1.0.0-beta.2', '<')) {
191-
console.error(`-- ${id}Skipping; Can only validate STAC version >= 1.0.0-beta.2\n`);
167+
else if (versions.compare(data.stac_version, '1.0.0-rc.1', '<')) {
168+
console.error(`-- ${id}Skipping; Can only validate STAC version >= 1.0.0-rc.1\n`);
192169
continue;
193170
}
194171
else if (verbose) {
195172
console.log(`-- ${id}STAC Version: ${data.stac_version}`);
196173
}
197174

198-
let type;
199-
if (data.type === 'Feature') {
200-
type = 'item';
201-
}
202-
else if (data.type === 'FeatureCollection') {
203-
// type = 'itemcollection';
204-
console.warn(`-- ${id}Skipping; STAC ItemCollections not supported yet\n`);
205-
continue;
206-
}
207-
else if (data.type === "Collection" || typeof data.extent !== 'undefined' || typeof data.license !== 'undefined') {
208-
type = 'collection';
209-
210-
}
211-
else if (data.type === "Catalog" || typeof data.description !== 'undefined') {
212-
type = 'catalog';
213-
}
214-
else {
215-
console.error(`-- ${id}Invalid; Can't detect which schema to use.\n`);
216-
fileValid = false;
217-
continue;
175+
switch(data.type) {
176+
case 'FeatureCollection':
177+
console.warn(`-- ${id}Skipping; STAC ItemCollections not supported yet\n`);
178+
continue;
179+
case 'Catalog':
180+
case 'Collection':
181+
case 'Feature':
182+
break;
183+
default:
184+
console.error(`-- ${id}Invalid; Can't detect type of the STAC object. Is the 'type' field missing or invalid?\n`);
185+
fileValid = false;
186+
continue;
218187
}
219188

220189
// Get all schema to validate against
221-
let schemas = [type];
190+
let schemas = [data.type];
222191
if (Array.isArray(data.stac_extensions)) {
223192
schemas = schemas.concat(data.stac_extensions);
193+
// Convert shortcuts supported in 1.0.0 RC1 into schema URLs
194+
if (versions.compare(data.stac_version, '1.0.0-rc.1', '=')) {
195+
schemas = schemas.map(ext => ext.replace(/^(eo|projection|scientific|view)$/, 'https://schemas.stacspec.org/v1.0.0-rc.1/extensions/$1/json-schema/schema.json'));
196+
}
224197
}
225198

226199
for(let schema of schemas) {
227200
try {
228-
let loadArgs = isUrl(schema) ? [schema] : [schemaFolder, data.stac_version, schema];
229-
let validate = await loadSchema(...loadArgs);
201+
let schemaId;
202+
let core = false;
203+
switch(schema) {
204+
case 'Feature':
205+
schema = 'Item';
206+
case 'Catalog':
207+
case 'Collection':
208+
let type = schema.toLowerCase();
209+
schemaId = `https://schemas.stacspec.org/v${data.stac_version}/${type}-spec/json-schema/${type}.json`;
210+
core = true;
211+
break;
212+
default: // extension
213+
if (isUrl(schema)) {
214+
schemaId = schema;
215+
}
216+
else {
217+
throw new Error("'stac_extensions' must contain a valid schema URL, not a shortcut.");
218+
}
219+
}
220+
let validate = await loadSchema(schemaId);
230221
let valid = validate(data);
231222
if (!valid) {
232223
console.log(`--- ${schema}: invalid`);
233224
console.warn(validate.errors);
234225
console.log("\n");
235226
fileValid = false;
236-
if (schema === 'core' && !DEBUG) {
227+
if (core && !DEBUG) {
237228
if (verbose) {
238229
console.info("-- Validation error in core, skipping extension validation");
239230
}
@@ -278,18 +269,24 @@ function matchFile(given, expected) {
278269
return normalizeNewline(given) === normalizeNewline(expected);
279270
}
280271

272+
function normalizePath(path) {
273+
return path.replace(/\\/g, '/').replace(/\/$/, "");
274+
}
275+
281276
function normalizeNewline(str) {
282277
// 2 spaces, *nix newlines, newline at end of file
283278
return str.trimRight().replace(/(\r\n|\r)/g, "\n") + "\n";
284279
}
285280

286281
function isUrl(uri) {
287-
let part = uri.match(/^(\w+):\/\//i);
288-
if(part) {
289-
if (!SUPPORTED_PROTOCOLS.includes(part[1].toLowerCase())) {
290-
throw new Error(`Given protocol "${part[1]}" is not supported.`);
282+
if (typeof uri === 'string') {
283+
let part = uri.match(/^(\w+):\/\//i);
284+
if(part) {
285+
if (!SUPPORTED_PROTOCOLS.includes(part[1].toLowerCase())) {
286+
throw new Error(`Given protocol "${part[1]}" is not supported.`);
287+
}
288+
return true;
291289
}
292-
return true;
293290
}
294291
return false;
295292
}
@@ -305,68 +302,43 @@ async function readExamples(folder) {
305302
return files;
306303
}
307304

308-
async function loadSchema(baseUrl = null, version = null, shortcut = null) {
309-
version = (typeof version === 'string') ? "v" + version : "unversioned";
310-
311-
if (typeof baseUrl !== 'string') {
312-
baseUrl = `https://schemas.stacspec.org/${version}`;
305+
async function loadJsonFromUri(uri) {
306+
if (schemaMap[uri]) {
307+
uri = schemaMap[uri];
313308
}
314-
else {
315-
baseUrl = baseUrl.replace(/\\/g, '/').replace(/\/$/, "");
316-
}
317-
318-
let url;
319-
let isExtension = false;
320-
if (shortcut === 'item' || shortcut === 'catalog' || shortcut === 'collection') {
321-
url = `${baseUrl}/${shortcut}-spec/json-schema/${shortcut}.json`;
309+
else if (schemaFolder) {
310+
uri = uri.replace(/^https:\/\/schemas\.stacspec\.org\/v[^\/]+/, schemaFolder);
322311
}
323-
else if (typeof shortcut === 'string') {
324-
if (shortcut === 'proj') {
325-
// Capture a very common mistake and give a better explanation (see #4)
326-
throw new Error("'stac_extensions' must contain 'projection instead of 'proj'.");
327-
}
328-
url = `${baseUrl}/extensions/${shortcut}/json-schema/schema.json`;
329-
isExtension = true;
312+
if (isUrl(uri)) {
313+
let response = await axios.get(uri);
314+
return response.data;
330315
}
331316
else {
332-
url = baseUrl;
317+
return JSON.parse(await fs.readFile(uri, "utf8"));
333318
}
319+
}
334320

335-
if (schemaMap[url]) {
336-
url = schemaMap[url];
321+
async function loadSchema(schemaId) {
322+
let schema = ajv.getSchema(schemaId);
323+
if (schema) {
324+
return schema;
337325
}
338326

339-
if (typeof COMPILED[url] !== 'undefined') {
340-
return COMPILED[url];
341-
}
342-
else {
343-
try {
344-
let parser = new $RefParser();
345-
let fullSchema = await parser.dereference(url, {
346-
dereference: {
347-
circular: 'ignore'
348-
}
349-
});
350-
COMPILED[url] = ajv.compile(fullSchema);
351-
if (parser.$refs.circular && verbose) {
352-
console.log(`--- Schema ${url} is circular, which is not supported by the library. Some properties may not get validated.`);
353-
}
354-
return COMPILED[url];
355-
} catch (error) {
356-
// Convert error to string, both for Error objects and strings
357-
let msg = "" + error;
358-
// Give better error message for (likely) invalid shortcuts
359-
if (isExtension && !SHORTCUTS.includes(shortcut) && (msg.includes("Error downloading") || msg.includes("Error opening file"))) {
360-
if (DEBUG) {
361-
console.trace(error);
362-
}
363-
throw new Error(`-- Schema at '${url}' not found. Please ensure all entries in 'stac_extensions' are valid.`);
364-
}
365-
else {
366-
throw error;
367-
}
327+
try {
328+
json = await loadJsonFromUri(schemaId);
329+
} catch (error) {
330+
if (DEBUG) {
331+
console.trace(error);
368332
}
333+
throw new Error(`-- Schema at '${schemaId}' not found. Please ensure all entries in 'stac_extensions' are valid.`);
369334
}
335+
336+
schema = ajv.getSchema(json.$id);
337+
if (schema) {
338+
return schema;
339+
}
340+
341+
return await ajv.compileAsync(json);
370342
}
371343

372344
module.exports = async () => {

‎package.json

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
22
"name": "stac-node-validator",
3-
"version": "1.0.1",
3+
"version": "1.1.0",
44
"description": "STAC Validator for NodeJS",
55
"author": "Matthias Mohr",
66
"license": "Apache-2.0",
@@ -26,9 +26,9 @@
2626
"index.js"
2727
],
2828
"dependencies": {
29-
"@apidevtools/json-schema-ref-parser": "^9.0.1",
3029
"ajv": "^6.12.2",
3130
"ajv-formats-draft2019": "^1.4.3",
31+
"axios": "^0.21.1",
3232
"compare-versions": "^3.6.0",
3333
"fs-extra": "^9.0.0",
3434
"jest-diff": "^26.6.2",

0 commit comments

Comments
 (0)