tab-yoinker initial commit
This commit is contained in:
Anonymous
commit
b207aaefc0
9
.gitignore
vendored
Normal file
9
.gitignore
vendored
Normal file
@ -0,0 +1,9 @@
|
||||
**/node_modules
|
||||
**/input/**/*.db
|
||||
**/input/**/*.db-journal
|
||||
**/input/**/*.json
|
||||
**/output/**/*.db
|
||||
**/output/**/*.db-journal
|
||||
**/output/**/*.json
|
||||
**/output/**/*.txt
|
||||
**/*.7z
|
||||
36
01-scraper-urls/01-scrape-bands.js
Normal file
36
01-scraper-urls/01-scrape-bands.js
Normal file
@ -0,0 +1,36 @@
|
||||
const Scraper = require('./url-scraper.js');
|
||||
|
||||
// Comment out the urls that you have already scraped
|
||||
const bandListUrls = [
|
||||
'https://www.ultimate-guitar.com/bands/0-9.htm',
|
||||
// 'https://www.ultimate-guitar.com/bands/a.htm',
|
||||
// 'https://www.ultimate-guitar.com/bands/b.htm',
|
||||
// 'https://www.ultimate-guitar.com/bands/c.htm',
|
||||
// 'https://www.ultimate-guitar.com/bands/d.htm',
|
||||
// 'https://www.ultimate-guitar.com/bands/e.htm',
|
||||
// 'https://www.ultimate-guitar.com/bands/f.htm',
|
||||
// 'https://www.ultimate-guitar.com/bands/g.htm',
|
||||
// 'https://www.ultimate-guitar.com/bands/h.htm',
|
||||
// 'https://www.ultimate-guitar.com/bands/i.htm',
|
||||
// 'https://www.ultimate-guitar.com/bands/j.htm',
|
||||
// 'https://www.ultimate-guitar.com/bands/k.htm',
|
||||
// 'https://www.ultimate-guitar.com/bands/l.htm',
|
||||
// 'https://www.ultimate-guitar.com/bands/m.htm',
|
||||
// 'https://www.ultimate-guitar.com/bands/n.htm',
|
||||
// 'https://www.ultimate-guitar.com/bands/o.htm',
|
||||
// 'https://www.ultimate-guitar.com/bands/p.htm',
|
||||
// 'https://www.ultimate-guitar.com/bands/q.htm',
|
||||
// 'https://www.ultimate-guitar.com/bands/r.htm',
|
||||
// 'https://www.ultimate-guitar.com/bands/s.htm',
|
||||
// 'https://www.ultimate-guitar.com/bands/t.htm',
|
||||
// 'https://www.ultimate-guitar.com/bands/u.htm',
|
||||
// 'https://www.ultimate-guitar.com/bands/v.htm',
|
||||
// 'https://www.ultimate-guitar.com/bands/w.htm',
|
||||
// 'https://www.ultimate-guitar.com/bands/x.htm',
|
||||
// 'https://www.ultimate-guitar.com/bands/y.htm',
|
||||
// 'https://www.ultimate-guitar.com/bands/z.htm',
|
||||
];
|
||||
|
||||
(async () => {
|
||||
await Scraper.scrapeBands(bandListUrls);
|
||||
})();
|
||||
43
01-scraper-urls/02-scrape-artist-tab-urls.js
Normal file
43
01-scraper-urls/02-scrape-artist-tab-urls.js
Normal file
@ -0,0 +1,43 @@
|
||||
const fs = require('fs/promises');
|
||||
const path = require('path');
|
||||
|
||||
const Scraper = require('./url-scraper.js');
|
||||
|
||||
// Comment out the artist files that you have already scraped
|
||||
const artistListFiles = [
|
||||
'output/artists/httpswww.ultimate-guitar.combands0-9.htm.json',
|
||||
// 'output/artists/httpswww.ultimate-guitar.combandsa.htm.json',
|
||||
// 'output/artists/httpswww.ultimate-guitar.combandsb.htm.json',
|
||||
// 'output/artists/httpswww.ultimate-guitar.combandsc.htm.json',
|
||||
// 'output/artists/httpswww.ultimate-guitar.combandsd.htm.json',
|
||||
// 'output/artists/httpswww.ultimate-guitar.combandse.htm.json',
|
||||
// 'output/artists/httpswww.ultimate-guitar.combandsf.htm.json',
|
||||
// 'output/artists/httpswww.ultimate-guitar.combandsg.htm.json',
|
||||
// 'output/artists/httpswww.ultimate-guitar.combandsh.htm.json',
|
||||
// 'output/artists/httpswww.ultimate-guitar.combandsi.htm.json',
|
||||
// 'output/artists/httpswww.ultimate-guitar.combandsj.htm.json',
|
||||
// 'output/artists/httpswww.ultimate-guitar.combandsk.htm.json',
|
||||
// 'output/artists/httpswww.ultimate-guitar.combandsl.htm.json',
|
||||
// 'output/artists/httpswww.ultimate-guitar.combandsm.htm.json',
|
||||
// 'output/artists/httpswww.ultimate-guitar.combandsn.htm.json',
|
||||
// 'output/artists/httpswww.ultimate-guitar.combandso.htm.json',
|
||||
// 'output/artists/httpswww.ultimate-guitar.combandsp.htm.json',
|
||||
// 'output/artists/httpswww.ultimate-guitar.combandsq.htm.json',
|
||||
// 'output/artists/httpswww.ultimate-guitar.combandsr.htm.json',
|
||||
// 'output/artists/httpswww.ultimate-guitar.combandss.htm.json',
|
||||
// 'output/artists/httpswww.ultimate-guitar.combandst.htm.json',
|
||||
// 'output/artists/httpswww.ultimate-guitar.combandsu.htm.json',
|
||||
// 'output/artists/httpswww.ultimate-guitar.combandsv.htm.json',
|
||||
// 'output/artists/httpswww.ultimate-guitar.combandsw.htm.json',
|
||||
// 'output/artists/httpswww.ultimate-guitar.combandsx.htm.json',
|
||||
// 'output/artists/httpswww.ultimate-guitar.combandsy.htm.json',
|
||||
// 'output/artists/httpswww.ultimate-guitar.combandsz.htm.json',
|
||||
];
|
||||
|
||||
(async () => {
|
||||
let num = 0;
|
||||
for (let file of artistListFiles) {
|
||||
let artists = JSON.parse(await fs.readFile(file));
|
||||
await Scraper.scrapeArtistTabUrls(path.join('output', 'artists-with-tabs', 'artists-part-' + num + '.json'), artists);
|
||||
}
|
||||
})();
|
||||
48
01-scraper-urls/concurrent-queue.js
Normal file
48
01-scraper-urls/concurrent-queue.js
Normal file
@ -0,0 +1,48 @@
|
||||
// Runs a limited number of promises at one time
|
||||
class ConcurrentQueue {
|
||||
constructor(consecutive) {
|
||||
this.consecutive = consecutive;
|
||||
this.queue = [];
|
||||
this.current = 0;
|
||||
this.drainListeners = [];
|
||||
}
|
||||
|
||||
_checkQueue() {
|
||||
if (this.current == 0 && this.queue.length == 0) {
|
||||
for (let drainListener of this.drainListeners) {
|
||||
drainListener();
|
||||
}
|
||||
this.drainListeners = [];
|
||||
}
|
||||
while (this.current < this.consecutive && this.queue.length > 0) {
|
||||
let taskData = this.queue.shift();
|
||||
this.current += 1;
|
||||
(async () => {
|
||||
try {
|
||||
taskData.resolve(await taskData.task());
|
||||
} catch (e) {
|
||||
taskData.reject(e);
|
||||
}
|
||||
this.current -= 1;
|
||||
this._checkQueue();
|
||||
})();
|
||||
}
|
||||
}
|
||||
|
||||
// returns a promise that can be awaited to get the resolution or rejection of the task's execution
|
||||
push(task) {
|
||||
return new Promise((resolve, reject) => {
|
||||
this.queue.push({ task, resolve, reject })
|
||||
this._checkQueue();
|
||||
});
|
||||
}
|
||||
|
||||
async waitForDrain() {
|
||||
return new Promise((resolve) => {
|
||||
this.drainListeners.push(resolve);
|
||||
this._checkQueue();
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = ConcurrentQueue;
|
||||
0
01-scraper-urls/output/artists-with-tabs/.keep
Normal file
0
01-scraper-urls/output/artists-with-tabs/.keep
Normal file
0
01-scraper-urls/output/artists/.keep
Normal file
0
01-scraper-urls/output/artists/.keep
Normal file
464
01-scraper-urls/package-lock.json
generated
Normal file
464
01-scraper-urls/package-lock.json
generated
Normal file
@ -0,0 +1,464 @@
|
||||
{
|
||||
"name": "01-scraper-urls",
|
||||
"version": "1.0.0",
|
||||
"lockfileVersion": 1,
|
||||
"requires": true,
|
||||
"dependencies": {
|
||||
"@tootallnate/once": {
|
||||
"version": "1.1.2",
|
||||
"resolved": "https://registry.npmjs.org/@tootallnate/once/-/once-1.1.2.tgz",
|
||||
"integrity": "sha512-RbzJvlNzmRq5c3O09UipeuXno4tA1FE6ikOjxZK0tuxVv3412l64l5t1W5pj4+rJq9vpkm/kwiR07aZXnsKPxw=="
|
||||
},
|
||||
"abab": {
|
||||
"version": "2.0.5",
|
||||
"resolved": "https://registry.npmjs.org/abab/-/abab-2.0.5.tgz",
|
||||
"integrity": "sha512-9IK9EadsbHo6jLWIpxpR6pL0sazTXV6+SQv25ZB+F7Bj9mJNaOc4nCRabwd5M/JwmUa8idz6Eci6eKfJryPs6Q=="
|
||||
},
|
||||
"acorn": {
|
||||
"version": "8.5.0",
|
||||
"resolved": "https://registry.npmjs.org/acorn/-/acorn-8.5.0.tgz",
|
||||
"integrity": "sha512-yXbYeFy+jUuYd3/CDcg2NkIYE991XYX/bje7LmjJigUciaeO1JR4XxXgCIV1/Zc/dRuFEyw1L0pbA+qynJkW5Q=="
|
||||
},
|
||||
"acorn-globals": {
|
||||
"version": "6.0.0",
|
||||
"resolved": "https://registry.npmjs.org/acorn-globals/-/acorn-globals-6.0.0.tgz",
|
||||
"integrity": "sha512-ZQl7LOWaF5ePqqcX4hLuv/bLXYQNfNWw2c0/yX/TsPRKamzHcTGQnlCjHT3TsmkOUVEPS3crCxiPfdzE/Trlhg==",
|
||||
"requires": {
|
||||
"acorn": "^7.1.1",
|
||||
"acorn-walk": "^7.1.1"
|
||||
},
|
||||
"dependencies": {
|
||||
"acorn": {
|
||||
"version": "7.4.1",
|
||||
"resolved": "https://registry.npmjs.org/acorn/-/acorn-7.4.1.tgz",
|
||||
"integrity": "sha512-nQyp0o1/mNdbTO1PO6kHkwSrmgZ0MT/jCCpNiwbUjGoRN4dlBhqJtoQuCnEOKzgTVwg0ZWiCoQy6SxMebQVh8A=="
|
||||
}
|
||||
}
|
||||
},
|
||||
"acorn-walk": {
|
||||
"version": "7.2.0",
|
||||
"resolved": "https://registry.npmjs.org/acorn-walk/-/acorn-walk-7.2.0.tgz",
|
||||
"integrity": "sha512-OPdCF6GsMIP+Az+aWfAAOEt2/+iVDKE7oy6lJ098aoe59oAmK76qV6Gw60SbZ8jHuG2wH058GF4pLFbYamYrVA=="
|
||||
},
|
||||
"agent-base": {
|
||||
"version": "6.0.2",
|
||||
"resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz",
|
||||
"integrity": "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==",
|
||||
"requires": {
|
||||
"debug": "4"
|
||||
}
|
||||
},
|
||||
"asynckit": {
|
||||
"version": "0.4.0",
|
||||
"resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz",
|
||||
"integrity": "sha1-x57Zf380y48robyXkLzDZkdLS3k="
|
||||
},
|
||||
"browser-process-hrtime": {
|
||||
"version": "1.0.0",
|
||||
"resolved": "https://registry.npmjs.org/browser-process-hrtime/-/browser-process-hrtime-1.0.0.tgz",
|
||||
"integrity": "sha512-9o5UecI3GhkpM6DrXr69PblIuWxPKk9Y0jHBRhdocZ2y7YECBFCsHm79Pr3OyR2AvjhDkabFJaDJMYRazHgsow=="
|
||||
},
|
||||
"combined-stream": {
|
||||
"version": "1.0.8",
|
||||
"resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz",
|
||||
"integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==",
|
||||
"requires": {
|
||||
"delayed-stream": "~1.0.0"
|
||||
}
|
||||
},
|
||||
"cssom": {
|
||||
"version": "0.5.0",
|
||||
"resolved": "https://registry.npmjs.org/cssom/-/cssom-0.5.0.tgz",
|
||||
"integrity": "sha512-iKuQcq+NdHqlAcwUY0o/HL69XQrUaQdMjmStJ8JFmUaiiQErlhrmuigkg/CU4E2J0IyUKUrMAgl36TvN67MqTw=="
|
||||
},
|
||||
"cssstyle": {
|
||||
"version": "2.3.0",
|
||||
"resolved": "https://registry.npmjs.org/cssstyle/-/cssstyle-2.3.0.tgz",
|
||||
"integrity": "sha512-AZL67abkUzIuvcHqk7c09cezpGNcxUxU4Ioi/05xHk4DQeTkWmGYftIE6ctU6AEt+Gn4n1lDStOtj7FKycP71A==",
|
||||
"requires": {
|
||||
"cssom": "~0.3.6"
|
||||
},
|
||||
"dependencies": {
|
||||
"cssom": {
|
||||
"version": "0.3.8",
|
||||
"resolved": "https://registry.npmjs.org/cssom/-/cssom-0.3.8.tgz",
|
||||
"integrity": "sha512-b0tGHbfegbhPJpxpiBPU2sCkigAqtM9O121le6bbOlgyV+NyGyCmVfJ6QW9eRjz8CpNfWEOYBIMIGRYkLwsIYg=="
|
||||
}
|
||||
}
|
||||
},
|
||||
"data-uri-to-buffer": {
|
||||
"version": "3.0.1",
|
||||
"resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-3.0.1.tgz",
|
||||
"integrity": "sha512-WboRycPNsVw3B3TL559F7kuBUM4d8CgMEvk6xEJlOp7OBPjt6G7z8WMWlD2rOFZLk6OYfFIUGsCOWzcQH9K2og=="
|
||||
},
|
||||
"data-urls": {
|
||||
"version": "3.0.0",
|
||||
"resolved": "https://registry.npmjs.org/data-urls/-/data-urls-3.0.0.tgz",
|
||||
"integrity": "sha512-4AefxbTTdFtxDUdh0BuMBs2qJVL25Mow2zlcuuePegQwgD6GEmQao42LLEeksOui8nL4RcNEugIpFP7eRd33xg==",
|
||||
"requires": {
|
||||
"abab": "^2.0.3",
|
||||
"whatwg-mimetype": "^2.3.0",
|
||||
"whatwg-url": "^9.0.0"
|
||||
}
|
||||
},
|
||||
"debug": {
|
||||
"version": "4.3.2",
|
||||
"resolved": "https://registry.npmjs.org/debug/-/debug-4.3.2.tgz",
|
||||
"integrity": "sha512-mOp8wKcvj7XxC78zLgw/ZA+6TSgkoE2C/ienthhRD298T7UNwAg9diBpLRxC0mOezLl4B0xV7M0cCO6P/O0Xhw==",
|
||||
"requires": {
|
||||
"ms": "2.1.2"
|
||||
}
|
||||
},
|
||||
"decimal.js": {
|
||||
"version": "10.3.1",
|
||||
"resolved": "https://registry.npmjs.org/decimal.js/-/decimal.js-10.3.1.tgz",
|
||||
"integrity": "sha512-V0pfhfr8suzyPGOx3nmq4aHqabehUZn6Ch9kyFpV79TGDTWFmHqUqXdabR7QHqxzrYolF4+tVmJhUG4OURg5dQ=="
|
||||
},
|
||||
"deep-is": {
|
||||
"version": "0.1.4",
|
||||
"resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.4.tgz",
|
||||
"integrity": "sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ=="
|
||||
},
|
||||
"delayed-stream": {
|
||||
"version": "1.0.0",
|
||||
"resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz",
|
||||
"integrity": "sha1-3zrhmayt+31ECqrgsp4icrJOxhk="
|
||||
},
|
||||
"domexception": {
|
||||
"version": "2.0.1",
|
||||
"resolved": "https://registry.npmjs.org/domexception/-/domexception-2.0.1.tgz",
|
||||
"integrity": "sha512-yxJ2mFy/sibVQlu5qHjOkf9J3K6zgmCxgJ94u2EdvDOV09H+32LtRswEcUsmUWN72pVLOEnTSRaIVVzVQgS0dg==",
|
||||
"requires": {
|
||||
"webidl-conversions": "^5.0.0"
|
||||
},
|
||||
"dependencies": {
|
||||
"webidl-conversions": {
|
||||
"version": "5.0.0",
|
||||
"resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-5.0.0.tgz",
|
||||
"integrity": "sha512-VlZwKPCkYKxQgeSbH5EyngOmRp7Ww7I9rQLERETtf5ofd9pGeswWiOtogpEO850jziPRarreGxn5QIiTqpb2wA=="
|
||||
}
|
||||
}
|
||||
},
|
||||
"escodegen": {
|
||||
"version": "2.0.0",
|
||||
"resolved": "https://registry.npmjs.org/escodegen/-/escodegen-2.0.0.tgz",
|
||||
"integrity": "sha512-mmHKys/C8BFUGI+MAWNcSYoORYLMdPzjrknd2Vc+bUsjN5bXcr8EhrNB+UTqfL1y3I9c4fw2ihgtMPQLBRiQxw==",
|
||||
"requires": {
|
||||
"esprima": "^4.0.1",
|
||||
"estraverse": "^5.2.0",
|
||||
"esutils": "^2.0.2",
|
||||
"optionator": "^0.8.1",
|
||||
"source-map": "~0.6.1"
|
||||
}
|
||||
},
|
||||
"esprima": {
|
||||
"version": "4.0.1",
|
||||
"resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz",
|
||||
"integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A=="
|
||||
},
|
||||
"estraverse": {
|
||||
"version": "5.2.0",
|
||||
"resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.2.0.tgz",
|
||||
"integrity": "sha512-BxbNGGNm0RyRYvUdHpIwv9IWzeM9XClbOxwoATuFdOE7ZE6wHL+HQ5T8hoPM+zHvmKzzsEqhgy0GrQ5X13afiQ=="
|
||||
},
|
||||
"esutils": {
|
||||
"version": "2.0.3",
|
||||
"resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz",
|
||||
"integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g=="
|
||||
},
|
||||
"fast-levenshtein": {
|
||||
"version": "2.0.6",
|
||||
"resolved": "https://registry.npmjs.org/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz",
|
||||
"integrity": "sha1-PYpcZog6FqMMqGQ+hR8Zuqd5eRc="
|
||||
},
|
||||
"fetch-blob": {
|
||||
"version": "3.1.2",
|
||||
"resolved": "https://registry.npmjs.org/fetch-blob/-/fetch-blob-3.1.2.tgz",
|
||||
"integrity": "sha512-hunJbvy/6OLjCD0uuhLdp0mMPzP/yd2ssd1t2FCJsaA7wkWhpbp9xfuNVpv7Ll4jFhzp6T4LAupSiV9uOeg0VQ==",
|
||||
"requires": {
|
||||
"web-streams-polyfill": "^3.0.3"
|
||||
}
|
||||
},
|
||||
"form-data": {
|
||||
"version": "4.0.0",
|
||||
"resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz",
|
||||
"integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==",
|
||||
"requires": {
|
||||
"asynckit": "^0.4.0",
|
||||
"combined-stream": "^1.0.8",
|
||||
"mime-types": "^2.1.12"
|
||||
}
|
||||
},
|
||||
"html-encoding-sniffer": {
|
||||
"version": "2.0.1",
|
||||
"resolved": "https://registry.npmjs.org/html-encoding-sniffer/-/html-encoding-sniffer-2.0.1.tgz",
|
||||
"integrity": "sha512-D5JbOMBIR/TVZkubHT+OyT2705QvogUW4IBn6nHd756OwieSF9aDYFj4dv6HHEVGYbHaLETa3WggZYWWMyy3ZQ==",
|
||||
"requires": {
|
||||
"whatwg-encoding": "^1.0.5"
|
||||
}
|
||||
},
|
||||
"http-proxy-agent": {
|
||||
"version": "4.0.1",
|
||||
"resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-4.0.1.tgz",
|
||||
"integrity": "sha512-k0zdNgqWTGA6aeIRVpvfVob4fL52dTfaehylg0Y4UvSySvOq/Y+BOyPrgpUrA7HylqvU8vIZGsRuXmspskV0Tg==",
|
||||
"requires": {
|
||||
"@tootallnate/once": "1",
|
||||
"agent-base": "6",
|
||||
"debug": "4"
|
||||
}
|
||||
},
|
||||
"https-proxy-agent": {
|
||||
"version": "5.0.0",
|
||||
"resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.0.tgz",
|
||||
"integrity": "sha512-EkYm5BcKUGiduxzSt3Eppko+PiNWNEpa4ySk9vTC6wDsQJW9rHSa+UhGNJoRYp7bz6Ht1eaRIa6QaJqO5rCFbA==",
|
||||
"requires": {
|
||||
"agent-base": "6",
|
||||
"debug": "4"
|
||||
}
|
||||
},
|
||||
"iconv-lite": {
|
||||
"version": "0.4.24",
|
||||
"resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz",
|
||||
"integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==",
|
||||
"requires": {
|
||||
"safer-buffer": ">= 2.1.2 < 3"
|
||||
}
|
||||
},
|
||||
"is-potential-custom-element-name": {
|
||||
"version": "1.0.1",
|
||||
"resolved": "https://registry.npmjs.org/is-potential-custom-element-name/-/is-potential-custom-element-name-1.0.1.tgz",
|
||||
"integrity": "sha512-bCYeRA2rVibKZd+s2625gGnGF/t7DSqDs4dP7CrLA1m7jKWz6pps0LpYLJN8Q64HtmPKJ1hrN3nzPNKFEKOUiQ=="
|
||||
},
|
||||
"jsdom": {
|
||||
"version": "17.0.0",
|
||||
"resolved": "https://registry.npmjs.org/jsdom/-/jsdom-17.0.0.tgz",
|
||||
"integrity": "sha512-MUq4XdqwtNurZDVeKScENMPHnkgmdIvMzZ1r1NSwHkDuaqI6BouPjr+17COo4/19oLNnmdpFDPOHVpgIZmZ+VA==",
|
||||
"requires": {
|
||||
"abab": "^2.0.5",
|
||||
"acorn": "^8.4.1",
|
||||
"acorn-globals": "^6.0.0",
|
||||
"cssom": "^0.5.0",
|
||||
"cssstyle": "^2.3.0",
|
||||
"data-urls": "^3.0.0",
|
||||
"decimal.js": "^10.3.1",
|
||||
"domexception": "^2.0.1",
|
||||
"escodegen": "^2.0.0",
|
||||
"form-data": "^4.0.0",
|
||||
"html-encoding-sniffer": "^2.0.1",
|
||||
"http-proxy-agent": "^4.0.1",
|
||||
"https-proxy-agent": "^5.0.0",
|
||||
"is-potential-custom-element-name": "^1.0.1",
|
||||
"nwsapi": "^2.2.0",
|
||||
"parse5": "6.0.1",
|
||||
"saxes": "^5.0.1",
|
||||
"symbol-tree": "^3.2.4",
|
||||
"tough-cookie": "^4.0.0",
|
||||
"w3c-hr-time": "^1.0.2",
|
||||
"w3c-xmlserializer": "^2.0.0",
|
||||
"webidl-conversions": "^6.1.0",
|
||||
"whatwg-encoding": "^1.0.5",
|
||||
"whatwg-mimetype": "^2.3.0",
|
||||
"whatwg-url": "^9.0.0",
|
||||
"ws": "^8.0.0",
|
||||
"xml-name-validator": "^3.0.0"
|
||||
}
|
||||
},
|
||||
"levn": {
|
||||
"version": "0.3.0",
|
||||
"resolved": "https://registry.npmjs.org/levn/-/levn-0.3.0.tgz",
|
||||
"integrity": "sha1-OwmSTt+fCDwEkP3UwLxEIeBHZO4=",
|
||||
"requires": {
|
||||
"prelude-ls": "~1.1.2",
|
||||
"type-check": "~0.3.2"
|
||||
}
|
||||
},
|
||||
"mime-db": {
|
||||
"version": "1.49.0",
|
||||
"resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.49.0.tgz",
|
||||
"integrity": "sha512-CIc8j9URtOVApSFCQIF+VBkX1RwXp/oMMOrqdyXSBXq5RWNEsRfyj1kiRnQgmNXmHxPoFIxOroKA3zcU9P+nAA=="
|
||||
},
|
||||
"mime-types": {
|
||||
"version": "2.1.32",
|
||||
"resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.32.tgz",
|
||||
"integrity": "sha512-hJGaVS4G4c9TSMYh2n6SQAGrC4RnfU+daP8G7cSCmaqNjiOoUY0VHCMS42pxnQmVF1GWwFhbHWn3RIxCqTmZ9A==",
|
||||
"requires": {
|
||||
"mime-db": "1.49.0"
|
||||
}
|
||||
},
|
||||
"ms": {
|
||||
"version": "2.1.2",
|
||||
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
|
||||
"integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w=="
|
||||
},
|
||||
"node-fetch": {
|
||||
"version": "3.0.0",
|
||||
"resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.0.0.tgz",
|
||||
"integrity": "sha512-bKMI+C7/T/SPU1lKnbQbwxptpCrG9ashG+VkytmXCPZyuM9jB6VU+hY0oi4lC8LxTtAeWdckNCTa3nrGsAdA3Q==",
|
||||
"requires": {
|
||||
"data-uri-to-buffer": "^3.0.1",
|
||||
"fetch-blob": "^3.1.2"
|
||||
}
|
||||
},
|
||||
"nwsapi": {
|
||||
"version": "2.2.0",
|
||||
"resolved": "https://registry.npmjs.org/nwsapi/-/nwsapi-2.2.0.tgz",
|
||||
"integrity": "sha512-h2AatdwYH+JHiZpv7pt/gSX1XoRGb7L/qSIeuqA6GwYoF9w1vP1cw42TO0aI2pNyshRK5893hNSl+1//vHK7hQ=="
|
||||
},
|
||||
"optionator": {
|
||||
"version": "0.8.3",
|
||||
"resolved": "https://registry.npmjs.org/optionator/-/optionator-0.8.3.tgz",
|
||||
"integrity": "sha512-+IW9pACdk3XWmmTXG8m3upGUJst5XRGzxMRjXzAuJ1XnIFNvfhjjIuYkDvysnPQ7qzqVzLt78BCruntqRhWQbA==",
|
||||
"requires": {
|
||||
"deep-is": "~0.1.3",
|
||||
"fast-levenshtein": "~2.0.6",
|
||||
"levn": "~0.3.0",
|
||||
"prelude-ls": "~1.1.2",
|
||||
"type-check": "~0.3.2",
|
||||
"word-wrap": "~1.2.3"
|
||||
}
|
||||
},
|
||||
"parse5": {
|
||||
"version": "6.0.1",
|
||||
"resolved": "https://registry.npmjs.org/parse5/-/parse5-6.0.1.tgz",
|
||||
"integrity": "sha512-Ofn/CTFzRGTTxwpNEs9PP93gXShHcTq255nzRYSKe8AkVpZY7e1fpmTfOyoIvjP5HG7Z2ZM7VS9PPhQGW2pOpw=="
|
||||
},
|
||||
"prelude-ls": {
|
||||
"version": "1.1.2",
|
||||
"resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.1.2.tgz",
|
||||
"integrity": "sha1-IZMqVJ9eUv/ZqCf1cOBL5iqX2lQ="
|
||||
},
|
||||
"psl": {
|
||||
"version": "1.8.0",
|
||||
"resolved": "https://registry.npmjs.org/psl/-/psl-1.8.0.tgz",
|
||||
"integrity": "sha512-RIdOzyoavK+hA18OGGWDqUTsCLhtA7IcZ/6NCs4fFJaHBDab+pDDmDIByWFRQJq2Cd7r1OoQxBGKOaztq+hjIQ=="
|
||||
},
|
||||
"punycode": {
|
||||
"version": "2.1.1",
|
||||
"resolved": "https://registry.npmjs.org/punycode/-/punycode-2.1.1.tgz",
|
||||
"integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A=="
|
||||
},
|
||||
"safer-buffer": {
|
||||
"version": "2.1.2",
|
||||
"resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz",
|
||||
"integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg=="
|
||||
},
|
||||
"saxes": {
|
||||
"version": "5.0.1",
|
||||
"resolved": "https://registry.npmjs.org/saxes/-/saxes-5.0.1.tgz",
|
||||
"integrity": "sha512-5LBh1Tls8c9xgGjw3QrMwETmTMVk0oFgvrFSvWx62llR2hcEInrKNZ2GZCCuuy2lvWrdl5jhbpeqc5hRYKFOcw==",
|
||||
"requires": {
|
||||
"xmlchars": "^2.2.0"
|
||||
}
|
||||
},
|
||||
"source-map": {
|
||||
"version": "0.6.1",
|
||||
"resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz",
|
||||
"integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==",
|
||||
"optional": true
|
||||
},
|
||||
"symbol-tree": {
|
||||
"version": "3.2.4",
|
||||
"resolved": "https://registry.npmjs.org/symbol-tree/-/symbol-tree-3.2.4.tgz",
|
||||
"integrity": "sha512-9QNk5KwDF+Bvz+PyObkmSYjI5ksVUYtjW7AU22r2NKcfLJcXp96hkDWU3+XndOsUb+AQ9QhfzfCT2O+CNWT5Tw=="
|
||||
},
|
||||
"tough-cookie": {
|
||||
"version": "4.0.0",
|
||||
"resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-4.0.0.tgz",
|
||||
"integrity": "sha512-tHdtEpQCMrc1YLrMaqXXcj6AxhYi/xgit6mZu1+EDWUn+qhUf8wMQoFIy9NXuq23zAwtcB0t/MjACGR18pcRbg==",
|
||||
"requires": {
|
||||
"psl": "^1.1.33",
|
||||
"punycode": "^2.1.1",
|
||||
"universalify": "^0.1.2"
|
||||
}
|
||||
},
|
||||
"tr46": {
|
||||
"version": "2.1.0",
|
||||
"resolved": "https://registry.npmjs.org/tr46/-/tr46-2.1.0.tgz",
|
||||
"integrity": "sha512-15Ih7phfcdP5YxqiB+iDtLoaTz4Nd35+IiAv0kQ5FNKHzXgdWqPoTIqEDDJmXceQt4JZk6lVPT8lnDlPpGDppw==",
|
||||
"requires": {
|
||||
"punycode": "^2.1.1"
|
||||
}
|
||||
},
|
||||
"type-check": {
|
||||
"version": "0.3.2",
|
||||
"resolved": "https://registry.npmjs.org/type-check/-/type-check-0.3.2.tgz",
|
||||
"integrity": "sha1-WITKtRLPHTVeP7eE8wgEsrUg23I=",
|
||||
"requires": {
|
||||
"prelude-ls": "~1.1.2"
|
||||
}
|
||||
},
|
||||
"universalify": {
|
||||
"version": "0.1.2",
|
||||
"resolved": "https://registry.npmjs.org/universalify/-/universalify-0.1.2.tgz",
|
||||
"integrity": "sha512-rBJeI5CXAlmy1pV+617WB9J63U6XcazHHF2f2dbJix4XzpUF0RS3Zbj0FGIOCAva5P/d/GBOYaACQ1w+0azUkg=="
|
||||
},
|
||||
"w3c-hr-time": {
|
||||
"version": "1.0.2",
|
||||
"resolved": "https://registry.npmjs.org/w3c-hr-time/-/w3c-hr-time-1.0.2.tgz",
|
||||
"integrity": "sha512-z8P5DvDNjKDoFIHK7q8r8lackT6l+jo/Ye3HOle7l9nICP9lf1Ci25fy9vHd0JOWewkIFzXIEig3TdKT7JQ5fQ==",
|
||||
"requires": {
|
||||
"browser-process-hrtime": "^1.0.0"
|
||||
}
|
||||
},
|
||||
"w3c-xmlserializer": {
|
||||
"version": "2.0.0",
|
||||
"resolved": "https://registry.npmjs.org/w3c-xmlserializer/-/w3c-xmlserializer-2.0.0.tgz",
|
||||
"integrity": "sha512-4tzD0mF8iSiMiNs30BiLO3EpfGLZUT2MSX/G+o7ZywDzliWQ3OPtTZ0PTC3B3ca1UAf4cJMHB+2Bf56EriJuRA==",
|
||||
"requires": {
|
||||
"xml-name-validator": "^3.0.0"
|
||||
}
|
||||
},
|
||||
"web-streams-polyfill": {
|
||||
"version": "3.1.1",
|
||||
"resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.1.1.tgz",
|
||||
"integrity": "sha512-Czi3fG883e96T4DLEPRvufrF2ydhOOW1+1a6c3gNjH2aIh50DNFBdfwh2AKoOf1rXvpvavAoA11Qdq9+BKjE0Q=="
|
||||
},
|
||||
"webidl-conversions": {
|
||||
"version": "6.1.0",
|
||||
"resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-6.1.0.tgz",
|
||||
"integrity": "sha512-qBIvFLGiBpLjfwmYAaHPXsn+ho5xZnGvyGvsarywGNc8VyQJUMHJ8OBKGGrPER0okBeMDaan4mNBlgBROxuI8w=="
|
||||
},
|
||||
"whatwg-encoding": {
|
||||
"version": "1.0.5",
|
||||
"resolved": "https://registry.npmjs.org/whatwg-encoding/-/whatwg-encoding-1.0.5.tgz",
|
||||
"integrity": "sha512-b5lim54JOPN9HtzvK9HFXvBma/rnfFeqsic0hSpjtDbVxR3dJKLc+KB4V6GgiGOvl7CY/KNh8rxSo9DKQrnUEw==",
|
||||
"requires": {
|
||||
"iconv-lite": "0.4.24"
|
||||
}
|
||||
},
|
||||
"whatwg-mimetype": {
|
||||
"version": "2.3.0",
|
||||
"resolved": "https://registry.npmjs.org/whatwg-mimetype/-/whatwg-mimetype-2.3.0.tgz",
|
||||
"integrity": "sha512-M4yMwr6mAnQz76TbJm914+gPpB/nCwvZbJU28cUD6dR004SAxDLOOSUaB1JDRqLtaOV/vi0IC5lEAGFgrjGv/g=="
|
||||
},
|
||||
"whatwg-url": {
|
||||
"version": "9.1.0",
|
||||
"resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-9.1.0.tgz",
|
||||
"integrity": "sha512-CQ0UcrPHyomtlOCot1TL77WyMIm/bCwrJ2D6AOKGwEczU9EpyoqAokfqrf/MioU9kHcMsmJZcg1egXix2KYEsA==",
|
||||
"requires": {
|
||||
"tr46": "^2.1.0",
|
||||
"webidl-conversions": "^6.1.0"
|
||||
}
|
||||
},
|
||||
"word-wrap": {
|
||||
"version": "1.2.3",
|
||||
"resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.3.tgz",
|
||||
"integrity": "sha512-Hz/mrNwitNRh/HUAtM/VT/5VH+ygD6DV7mYKZAtHOrbs8U7lvPS6xf7EJKMF0uW1KJCl0H701g3ZGus+muE5vQ=="
|
||||
},
|
||||
"ws": {
|
||||
"version": "8.2.2",
|
||||
"resolved": "https://registry.npmjs.org/ws/-/ws-8.2.2.tgz",
|
||||
"integrity": "sha512-Q6B6H2oc8QY3llc3cB8kVmQ6pnJWVQbP7Q5algTcIxx7YEpc0oU4NBVHlztA7Ekzfhw2r0rPducMUiCGWKQRzw=="
|
||||
},
|
||||
"xml-name-validator": {
|
||||
"version": "3.0.0",
|
||||
"resolved": "https://registry.npmjs.org/xml-name-validator/-/xml-name-validator-3.0.0.tgz",
|
||||
"integrity": "sha512-A5CUptxDsvxKJEU3yO6DuWBSJz/qizqzJKOMIfUJHETbBw/sFaDxgd6fxm1ewUaM0jZ444Fc5vC5ROYurg/4Pw=="
|
||||
},
|
||||
"xmlchars": {
|
||||
"version": "2.2.0",
|
||||
"resolved": "https://registry.npmjs.org/xmlchars/-/xmlchars-2.2.0.tgz",
|
||||
"integrity": "sha512-JZnDKK8B0RCDw84FNdDAIpZK+JuJw+s7Lz8nksI7SIuU3UXJJslUthsi+uWBUYOwPFwW7W7PRLRfUKpxjtjFCw=="
|
||||
}
|
||||
}
|
||||
}
|
||||
15
01-scraper-urls/package.json
Normal file
15
01-scraper-urls/package.json
Normal file
@ -0,0 +1,15 @@
|
||||
{
|
||||
"name": "01-scraper-urls",
|
||||
"version": "1.0.0",
|
||||
"description": "",
|
||||
"main": "01-scrape-bands.js",
|
||||
"scripts": {
|
||||
"test": "echo \"Error: no test specified\" && exit 1"
|
||||
},
|
||||
"author": "",
|
||||
"license": "ISC",
|
||||
"dependencies": {
|
||||
"jsdom": "^17.0.0",
|
||||
"node-fetch": "^3.0.0"
|
||||
}
|
||||
}
|
||||
255
01-scraper-urls/url-scraper.js
Normal file
255
01-scraper-urls/url-scraper.js
Normal file
@ -0,0 +1,255 @@
|
||||
const fs = require('fs/promises');
|
||||
const path = require('path');
|
||||
|
||||
const jsdom = require('jsdom');
|
||||
const fetch = (...args) => import('node-fetch').then(({default: fetch}) => fetch(...args));
|
||||
|
||||
const ConcurrentQueue = require('./concurrent-queue.js');
|
||||
|
||||
class ScrapeError extends Error {
|
||||
constructor(message, options, fileName, lineNumber) {
|
||||
super(...arguments);
|
||||
this.name = 'ScrapeError';
|
||||
}
|
||||
}
|
||||
|
||||
// From Cordis util.js
|
||||
function sanitizeFileName(name) {
|
||||
// Windows Version (created for Windows, most likely works cross-platform too given my research)
|
||||
// Allowed Characters: Extended Unicode Charset (1-255)
|
||||
// Illegal file names: CON, PRN, AUX, NUL, COM1, COM2, ..., COM9, LPT1, LPT2, ..., LPT9
|
||||
// Reserved Characters: <>:"/\|?*
|
||||
// Solution: Replace reserved characters with empty string (''), bad characters with '_', and append '_' to bad names
|
||||
|
||||
// Illegal File Names (Windows)
|
||||
if ([ 'CON', 'PRN', 'AUX', 'NUL',
|
||||
'COM1', 'COM2', 'COM3', 'COM4', 'COM5', 'COM6', 'COM7', 'COM8', 'COM9',
|
||||
'LPT1', 'LPT2', 'LPT3', 'LPT4', 'LPT5', 'LPT6', 'LPT7', 'LPT8', 'LPT9' ].indexOf(name) != -1) { // TODO: case insensitive?
|
||||
name += '_';
|
||||
}
|
||||
// Reserved Characters
|
||||
name = name.replace(/[<>:\"\/\\|?*]/g, '');
|
||||
// Allowed Characters
|
||||
return name.split('').map(c => c.charCodeAt(0) < 255 && c.charCodeAt(0) > 0 ? c : '_').join('');
|
||||
|
||||
// Much stricter whitelist version
|
||||
// replace bad characters with '_'
|
||||
//return name.split('').map(c => /[A-Za-z0-9-]/.exec(c) ? c : '_').join('');
|
||||
}
|
||||
|
||||
function estMSRemaining(startTime, ratioComplete) {
|
||||
return (1 - ratioComplete) * ((Date.now() - startTime) / ratioComplete);
|
||||
}
|
||||
|
||||
function formatRelative(msRelative) {
|
||||
if (msRelative < 1000) return `${(msRelative).toFixed(2)}ms`;
|
||||
else if (msRelative < 60 * 1000) return `${(msRelative / 1000).toFixed(2)}s`;
|
||||
else if (msRelative < 60 * 60 * 1000) return `${(msRelative / (60 * 1000)).toFixed(2)} mins`;
|
||||
else return `${(msRelative / (60 * 60 * 1000)).toFixed(2)} hours`;
|
||||
}
|
||||
|
||||
async function sleep(ms) {
|
||||
return new Promise((resolve) => {
|
||||
setTimeout(resolve, ms);
|
||||
});
|
||||
}
|
||||
|
||||
async function fuzzyDelay() {
|
||||
await sleep(500 + (500 * Math.random()));
|
||||
}
|
||||
|
||||
async function saveJsonData(filename, dataJSON) {
|
||||
await fs.writeFile(filename, JSON.stringify(dataJSON));
|
||||
}
|
||||
|
||||
// Note: This is the key scraper function. It scrapes the .js-store's data
|
||||
async function scrapeUGDataContent(url) {
|
||||
let page = null;
|
||||
try {
|
||||
page = await fetch(url);
|
||||
} catch (e) {
|
||||
throw new ScrapeError('Unable to fetch url', { cause: e });
|
||||
}
|
||||
|
||||
let text = null;
|
||||
try {
|
||||
text = await page.text();
|
||||
} catch (e) {
|
||||
throw new ScrapeError('Unable to decode page', { cause: e });
|
||||
}
|
||||
|
||||
let dom = null;
|
||||
try {
|
||||
dom = new jsdom.JSDOM(text);
|
||||
} catch (e) {
|
||||
throw new ScrapeError('Unable to parse document', { cause: e });
|
||||
}
|
||||
|
||||
if (!dom.window || !dom.window.document) {
|
||||
throw new ScrapeError('Unable to parse document');
|
||||
}
|
||||
|
||||
let document = dom.window.document;
|
||||
|
||||
const jsStore = document.querySelector('.js-store');
|
||||
if (jsStore == null) {
|
||||
throw new ScrapeError('Unable to find .js-store element for ' + url);
|
||||
}
|
||||
|
||||
const contentJSON = jsStore.getAttribute('data-content');
|
||||
if (contentJSON == null) {
|
||||
throw new ScrapeError('Unable to find data-content attribute on .js-store');
|
||||
}
|
||||
|
||||
const content = JSON.parse(contentJSON);
|
||||
return content;
|
||||
}
|
||||
|
||||
function parseTab(ugDataContent) {
|
||||
const store = ugDataContent.store; if (store === null) throw new ScrapeError('Unable to get ugDataContent.store');
|
||||
const page = store.page; if (page === null) throw new ScrapeError('Unable to get ugDataContent.store.page');
|
||||
const data = page.data; if (data === null) throw new ScrapeError('Unable to get ugDataContent.store.page.data');
|
||||
const meta = data.tab; if (meta === null) throw new ScrapeError('Unable to get ugDataContent.store.page.data.tab');
|
||||
const tview = data.tab_view; if (tview === null) throw new ScrapeError('Unable to get ugDataContent.store.page.data.tab_view');
|
||||
const wktab = tview.wiki_tab; if (wktab === null) throw new ScrapeError('Unable to get ugDataContent.store.page.data.tab_view.wiki_tab');
|
||||
const text = wktab.content; if (text === null) throw new ScrapeError('Unable to get ugDataContent.store.page.data.tab_view.wiki_tab.content');
|
||||
return { meta, text };
|
||||
}
|
||||
|
||||
function parseBandsPage(ugDataContent) {
|
||||
const store = ugDataContent.store; if (store === null) throw new ScrapeError('Unable to get ugDataContent.store');
|
||||
const page = store.page; if (page === null) throw new ScrapeError('Unable to get ugDataContent.store.page');
|
||||
const data = page.data; if (data === null) throw new ScrapeError('Unable to get ugDataContent.store.page.data');
|
||||
const alpha = data.alpha; if (alpha === null) throw new ScrapeError('Unable to get ugDataContent.store.page.data.alpha');
|
||||
const artists = data.artists; if (artists === null) throw new ScrapeError('Unable to get ugDataContent.store.page.data.artists');
|
||||
const pagenum = data.current_page; if (pagenum === null) throw new ScrapeError('Unable to get ugDataContent.store.page.data.current_page');
|
||||
const pagecnt = data.page_count; if (pagecnt === null) throw new ScrapeError('Unable to get ugDataContent.store.page.data.page_count');
|
||||
return { alpha, artists, pagenum, pagecnt };
|
||||
}
|
||||
|
||||
function parseArtistPage(ugDataContent) {
|
||||
const store = ugDataContent.store; if (store === null) throw new ScrapeError('Unable to get ugDataContent.store');
|
||||
const page = store.page; if (page === null) throw new ScrapeError('Unable to get ugDataContent.store.page');
|
||||
const data = page.data; if (data === null) throw new ScrapeError('Unable to get ugDataContent.store.page.data');
|
||||
const pagination = data.pagination; if (pagination === null) throw new ScrapeError('Unable to get ugDataContent.store.page.data.pagination');
|
||||
const pagenum = pagination.current; if (pagenum === null) throw new ScrapeError('Unable to get ugDataContent.store.page.data.pagination.current');
|
||||
const pages = pagination.pages; if (pages === null) throw new ScrapeError('Unable to get ugDataContent.store.page.data.pagination.pages');
|
||||
|
||||
const albumTabs = data.album_tabs; if (albumTabs === null) throw new ScrapeError('Unable to get ugDataContent.store.page.data.album_tabs');
|
||||
const chordProTabs = data.chord_pro_tabs; if (chordProTabs === null) throw new ScrapeError('Unable to get ugDataContent.store.page.data.chord_pro_tabs');
|
||||
const featTabs = data.feat_tabs; if (chordProTabs === null) throw new ScrapeError('Unable to get ugDataContent.store.page.data.feat_tabs');
|
||||
const otherTabs = data.other_tabs; if (chordProTabs === null) throw new ScrapeError('Unable to get ugDataContent.store.page.data.other_tabs');
|
||||
|
||||
return { albumTabs, chordProTabs, featTabs, otherTabs, pagenum, pages };
|
||||
}
|
||||
|
||||
// Returns a list of tab metadata (including tab URL)
|
||||
async function scrapeAllArtistTabListPages(startURL) {
|
||||
let tabs = [];
|
||||
let url = new URL(startURL); // Note: not considering the <base> tag, would have to change the implementation if this gets used somewhere.
|
||||
while (true) {
|
||||
//console.log('scraping artist page: ' + url.toString());
|
||||
const ugDataContent = await scrapeUGDataContent(url.toString());
|
||||
const page = parseArtistPage(ugDataContent);
|
||||
|
||||
tabs = tabs.concat(page.albumTabs, page.chordProTabs, page.featTabs, page.otherTabs);
|
||||
|
||||
const nextPageData = page.pages.find(pageData => pageData.page == page.pagenum + 1);
|
||||
if (nextPageData == null) break;
|
||||
url = new URL(nextPageData.url, url);
|
||||
|
||||
await fuzzyDelay();
|
||||
}
|
||||
|
||||
// the autists at ug.com thought it would be a good idea to return the same tab (same id) on different pages. This filters out duplicates
|
||||
const uniqueTabIds = new Set();
|
||||
const uniqueTabs = [];
|
||||
for (let tab of tabs) {
|
||||
if (uniqueTabIds.has(tab.id)) continue;
|
||||
uniqueTabIds.add(tab.id);
|
||||
uniqueTabs.push(tab);
|
||||
}
|
||||
|
||||
return uniqueTabs;
|
||||
}
|
||||
|
||||
// Returns a list of artist metadata (including artist tab list URL)
|
||||
async function scrapeAllBandListPages(startURL) {
|
||||
let artists = [];
|
||||
|
||||
// https://www.ultimate-guitar.com/bands/d.htm
|
||||
let url = new URL(startURL);
|
||||
let startTime = Date.now();
|
||||
|
||||
while (true) {
|
||||
const ugDataContent = await scrapeUGDataContent(url.toString());
|
||||
const page = parseBandsPage(ugDataContent);
|
||||
|
||||
artists = artists.concat(page.artists);
|
||||
|
||||
let fromNow = formatRelative(estMSRemaining(startTime, page.pagenum / page.pagecnt));
|
||||
console.log(`Band List Status: ${page.pagenum} / ${page.pagecnt} pages complete (${(page.pagenum / page.pagecnt * 100).toFixed(2)}%, ${fromNow} remaining)`);
|
||||
|
||||
if (page.pagenum + 1 > page.pagecnt) break;
|
||||
url = new URL(startURL.slice(0, -4) + (page.pagenum + 1) + '.htm'); // d.htm (start) -> d2.htm -> d3.htm -> ...
|
||||
|
||||
await fuzzyDelay();
|
||||
}
|
||||
|
||||
return artists;
|
||||
}
|
||||
|
||||
async function saveBandList(filename, url) {
|
||||
let artists = await scrapeAllBandListPages(url);
|
||||
await saveJsonData(filename, artists);
|
||||
}
|
||||
|
||||
// nigger :)
|
||||
|
||||
async function saveBandLists(urls) {
|
||||
let startTime = Date.now();
|
||||
let completed = 0;
|
||||
for (const url of urls) {
|
||||
console.log('doing band list: ' + url);
|
||||
await saveBandList(path.join('output', 'artists', sanitizeFileName(url + '.json')), url);
|
||||
|
||||
completed += 1;
|
||||
let fromNow = formatRelative(estMSRemaining(startTime, completed / urls.length));
|
||||
console.log(`Save All Band List Status: ${completed} / ${urls.length} band lists complete (${(completed / urls.length * 100).toFixed(2)}%, ${fromNow} remaining)`);
|
||||
|
||||
await fuzzyDelay();
|
||||
}
|
||||
}
|
||||
|
||||
// Note: modifies artists to add a 'tabs' property to each artist. This property contains a list
|
||||
// of the artist's tab metadatas (tab text is done in a different step)
|
||||
async function saveArtistsWithTabMetadata(filename, artists) {
|
||||
const baseURL = 'https://www.ultimate-guitar.com/';
|
||||
|
||||
let startTime = Date.now();
|
||||
let completed = 0;
|
||||
let taskQueue = new ConcurrentQueue(8); // Run a maximum of 4 artist tab list scrapers at a time
|
||||
// Note: the concurrent queue will (almost certainly) cause the artists to be somewhat to completely out of order in the output
|
||||
for (let artist of artists) {
|
||||
taskQueue.push(async () => {
|
||||
let artistStartURL = new URL(artist.artist_url, baseURL);
|
||||
let artistTabs = await scrapeAllArtistTabListPages(artistStartURL);
|
||||
artist.tabs = artistTabs;
|
||||
|
||||
completed += 1;
|
||||
let fromNow = formatRelative(estMSRemaining(startTime, completed / artists.length));
|
||||
let pctPerMin = ((100 * completed / artists.length) / ((Date.now() - startTime) / (60 * 1000))).toFixed(2);
|
||||
let artistsPerMin = (completed / ((Date.now() - startTime) / (60 * 1000))).toFixed(2);
|
||||
console.log(`Save Artists with Tab Metadata Status: ${completed} / ${artists.length} artists complete (${(completed / artists.length * 100).toFixed(2)}%, ${fromNow} remaining, ${pctPerMin} %/min, ${artistsPerMin} artists/min)`);
|
||||
});
|
||||
}
|
||||
|
||||
await taskQueue.waitForDrain();
|
||||
|
||||
await saveJsonData(filename, artists);
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
scrapeBands: saveBandLists,
|
||||
scrapeArtistTabUrls: saveArtistsWithTabMetadata
|
||||
};
|
||||
126
02-ingest-sqlite/01-injest-sqlite.js
Normal file
126
02-ingest-sqlite/01-injest-sqlite.js
Normal file
@ -0,0 +1,126 @@
|
||||
// Injests from ./input/* into ./output/tabs.db
|
||||
|
||||
const sqlite3 = require('sqlite3');
|
||||
const sqlite = require('sqlite');
|
||||
|
||||
const fs = require('fs/promises');
|
||||
|
||||
(async () => {
|
||||
const db = await sqlite.open({
|
||||
driver: sqlite3.Database,
|
||||
filename: './output/tabs-no-text.db'
|
||||
});
|
||||
|
||||
await db.run(`
|
||||
CREATE TABLE IF NOT EXISTS artists (
|
||||
scrape_id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT
|
||||
, id INTEGER
|
||||
, name TEXT
|
||||
, tabscount INTEGER
|
||||
, artist_url TEXT
|
||||
, tabs_last_update_timestamp INTEGER
|
||||
)
|
||||
`);
|
||||
|
||||
await db.run(`
|
||||
CREATE TABLE IF NOT EXISTS tabs (
|
||||
scrape_id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT
|
||||
, artist_scrape_id INTEGER NOT NULL
|
||||
, id INTEGER
|
||||
, song_id INTEGER
|
||||
, song_name TEXT
|
||||
, artist_id INTEGER
|
||||
, artist_name INTEGER
|
||||
, type TEXT
|
||||
, part TEXT
|
||||
, version INTEGER
|
||||
, votes INTEGER
|
||||
, rating NUMERIC
|
||||
, date TEXT
|
||||
, status TEXT
|
||||
, preset_id INTEGER
|
||||
, tab_access_type TEXT
|
||||
, tp_version INTEGER
|
||||
, tonality_name TEXT
|
||||
, version_description TEXT
|
||||
, verified INTEGER
|
||||
, artist_url TEXT
|
||||
, tab_url TEXT
|
||||
, tab_text TEXT
|
||||
, difficulty TEXT
|
||||
, tuning TEXT
|
||||
, type_name TEXT
|
||||
, FOREIGN KEY (artist_scrape_id) REFERENCES artists(scrape_id)
|
||||
)
|
||||
`);
|
||||
|
||||
// Clear out the database
|
||||
await db.run('DELETE FROM tabs');
|
||||
await db.run('DELETE FROM artists');
|
||||
|
||||
const files = await fs.readdir('./input/');
|
||||
|
||||
const stmtAddArtist = await db.prepare(`
|
||||
INSERT INTO artists (
|
||||
id, name, tabscount, artist_url, tabs_last_update_timestamp
|
||||
) VALUES (
|
||||
?1, ?2, ?3, ?4, ?5
|
||||
)
|
||||
`);
|
||||
const stmtAddTab = await db.prepare(`
|
||||
INSERT INTO tabs (
|
||||
artist_scrape_id
|
||||
, id, song_id, song_name, artist_id, artist_name, type, part
|
||||
, version, votes, rating, date, status, preset_id, tab_access_type, tp_version, tonality_name
|
||||
, version_description, verified, artist_url, tab_url, tab_text, difficulty, tuning, type_name
|
||||
) VALUES (
|
||||
?1
|
||||
, ?2, ?3, ?4, ?5, ?6, ?7, ?8
|
||||
, ?9, ?10, ?11, ?12, ?13, ?14, ?15, ?16, ?17
|
||||
, ?18, ?19, ?20, ?21, ?22, ?23, ?24, ?25
|
||||
)
|
||||
`);
|
||||
|
||||
function addArtist(id, name, tabscount, artist_url, tabs_last_update_timestamp) {
|
||||
return stmtAddArtist.run([id, name, tabscount, artist_url, tabs_last_update_timestamp]);
|
||||
}
|
||||
|
||||
function addTab(
|
||||
artist_scrape_id, id, song_id, song_name, artist_id, artist_name, type, part, version, votes,
|
||||
rating, date, status, preset_id, tab_access_type, tp_version, tonality_name, version_description,
|
||||
verified, artist_url, tab_url, tab_text, difficulty, tuning, type_name
|
||||
) {
|
||||
return stmtAddTab.run([
|
||||
artist_scrape_id, id, song_id, song_name, artist_id, artist_name, type, part, version, votes,
|
||||
rating, date, status, preset_id, tab_access_type, tp_version, tonality_name, version_description,
|
||||
verified, artist_url, tab_url, tab_text, difficulty, tuning, type_name
|
||||
]);
|
||||
}
|
||||
|
||||
for (let file of files) {
|
||||
if (!file.endsWith('.json')) continue; // skip the .keep file
|
||||
console.log('reading ./input/' + file);
|
||||
let dataJSON = await fs.readFile('./input/' + file);
|
||||
let data = JSON.parse(dataJSON);
|
||||
let artistIndex = 0;
|
||||
for (let artist of data) {
|
||||
console.log(`adding artist (${artistIndex+1}/${data.length}, ${artist.tabs.length} tabs): ${artist.name}`)
|
||||
let artistResult = await addArtist(artist.id, artist.name, artist.tabscount, artist.artist_url, artist.tabs_last_update_timestamp);
|
||||
let artistScrapeId = artistResult.lastID;
|
||||
for (let tab of artist.tabs) {
|
||||
addTab(
|
||||
artistScrapeId,
|
||||
tab.id, tab.song_id, tab.song_name, tab.artist_id, tab.artist_name, tab.type, tab.part, tab.version, tab.votes,
|
||||
tab.rating, tab.date, tab.status, tab.preset_id, tab.tab_access_type, tab.tp_version, tab.tonality_name, tab.version_description,
|
||||
tab.verified, tab.artist_url, tab.tab_url, null, tab.difficulty, tab.tuning, tab.type_name
|
||||
);
|
||||
}
|
||||
artistIndex += 1;
|
||||
}
|
||||
}
|
||||
|
||||
await stmtAddTab.finalize();
|
||||
await stmtAddArtist.finalize();
|
||||
|
||||
await db.close();
|
||||
})();
|
||||
0
02-ingest-sqlite/input/.keep
Normal file
0
02-ingest-sqlite/input/.keep
Normal file
0
02-ingest-sqlite/output/.keep
Normal file
0
02-ingest-sqlite/output/.keep
Normal file
956
02-ingest-sqlite/package-lock.json
generated
Normal file
956
02-ingest-sqlite/package-lock.json
generated
Normal file
@ -0,0 +1,956 @@
|
||||
{
|
||||
"name": "02-ingest-sqlite",
|
||||
"version": "1.0.0",
|
||||
"lockfileVersion": 1,
|
||||
"requires": true,
|
||||
"dependencies": {
|
||||
"abbrev": {
|
||||
"version": "1.1.1",
|
||||
"resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz",
|
||||
"integrity": "sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q=="
|
||||
},
|
||||
"ajv": {
|
||||
"version": "6.12.6",
|
||||
"resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz",
|
||||
"integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==",
|
||||
"optional": true,
|
||||
"requires": {
|
||||
"fast-deep-equal": "^3.1.1",
|
||||
"fast-json-stable-stringify": "^2.0.0",
|
||||
"json-schema-traverse": "^0.4.1",
|
||||
"uri-js": "^4.2.2"
|
||||
}
|
||||
},
|
||||
"ansi-regex": {
|
||||
"version": "2.1.1",
|
||||
"resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz",
|
||||
"integrity": "sha1-w7M6te42DYbg5ijwRorn7yfWVN8="
|
||||
},
|
||||
"aproba": {
|
||||
"version": "1.2.0",
|
||||
"resolved": "https://registry.npmjs.org/aproba/-/aproba-1.2.0.tgz",
|
||||
"integrity": "sha512-Y9J6ZjXtoYh8RnXVCMOU/ttDmk1aBjunq9vO0ta5x85WDQiQfUF9sIPBITdbiiIVcBo03Hi3jMxigBtsddlXRw=="
|
||||
},
|
||||
"are-we-there-yet": {
|
||||
"version": "1.1.7",
|
||||
"resolved": "https://registry.npmjs.org/are-we-there-yet/-/are-we-there-yet-1.1.7.tgz",
|
||||
"integrity": "sha512-nxwy40TuMiUGqMyRHgCSWZ9FM4VAoRP4xUYSTv5ImRog+h9yISPbVH7H8fASCIzYn9wlEv4zvFL7uKDMCFQm3g==",
|
||||
"requires": {
|
||||
"delegates": "^1.0.0",
|
||||
"readable-stream": "^2.0.6"
|
||||
}
|
||||
},
|
||||
"asn1": {
|
||||
"version": "0.2.4",
|
||||
"resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.4.tgz",
|
||||
"integrity": "sha512-jxwzQpLQjSmWXgwaCZE9Nz+glAG01yF1QnWgbhGwHI5A6FRIEY6IVqtHhIepHqI7/kyEyQEagBC5mBEFlIYvdg==",
|
||||
"optional": true,
|
||||
"requires": {
|
||||
"safer-buffer": "~2.1.0"
|
||||
}
|
||||
},
|
||||
"assert-plus": {
|
||||
"version": "1.0.0",
|
||||
"resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz",
|
||||
"integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU=",
|
||||
"optional": true
|
||||
},
|
||||
"asynckit": {
|
||||
"version": "0.4.0",
|
||||
"resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz",
|
||||
"integrity": "sha1-x57Zf380y48robyXkLzDZkdLS3k=",
|
||||
"optional": true
|
||||
},
|
||||
"aws-sign2": {
|
||||
"version": "0.7.0",
|
||||
"resolved": "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.7.0.tgz",
|
||||
"integrity": "sha1-tG6JCTSpWR8tL2+G1+ap8bP+dqg=",
|
||||
"optional": true
|
||||
},
|
||||
"aws4": {
|
||||
"version": "1.11.0",
|
||||
"resolved": "https://registry.npmjs.org/aws4/-/aws4-1.11.0.tgz",
|
||||
"integrity": "sha512-xh1Rl34h6Fi1DC2WWKfxUTVqRsNnr6LsKz2+hfwDxQJWmrx8+c7ylaqBMcHfl1U1r2dsifOvKX3LQuLNZ+XSvA==",
|
||||
"optional": true
|
||||
},
|
||||
"balanced-match": {
|
||||
"version": "1.0.2",
|
||||
"resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz",
|
||||
"integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw=="
|
||||
},
|
||||
"bcrypt-pbkdf": {
|
||||
"version": "1.0.2",
|
||||
"resolved": "https://registry.npmjs.org/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz",
|
||||
"integrity": "sha1-pDAdOJtqQ/m2f/PKEaP2Y342Dp4=",
|
||||
"optional": true,
|
||||
"requires": {
|
||||
"tweetnacl": "^0.14.3"
|
||||
}
|
||||
},
|
||||
"block-stream": {
|
||||
"version": "0.0.9",
|
||||
"resolved": "https://registry.npmjs.org/block-stream/-/block-stream-0.0.9.tgz",
|
||||
"integrity": "sha1-E+v+d4oDIFz+A3UUgeu0szAMEmo=",
|
||||
"optional": true,
|
||||
"requires": {
|
||||
"inherits": "~2.0.0"
|
||||
}
|
||||
},
|
||||
"brace-expansion": {
|
||||
"version": "1.1.11",
|
||||
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz",
|
||||
"integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==",
|
||||
"requires": {
|
||||
"balanced-match": "^1.0.0",
|
||||
"concat-map": "0.0.1"
|
||||
}
|
||||
},
|
||||
"caseless": {
|
||||
"version": "0.12.0",
|
||||
"resolved": "https://registry.npmjs.org/caseless/-/caseless-0.12.0.tgz",
|
||||
"integrity": "sha1-G2gcIf+EAzyCZUMJBolCDRhxUdw=",
|
||||
"optional": true
|
||||
},
|
||||
"chownr": {
|
||||
"version": "1.1.4",
|
||||
"resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz",
|
||||
"integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg=="
|
||||
},
|
||||
"code-point-at": {
|
||||
"version": "1.1.0",
|
||||
"resolved": "https://registry.npmjs.org/code-point-at/-/code-point-at-1.1.0.tgz",
|
||||
"integrity": "sha1-DQcLTQQ6W+ozovGkDi7bPZpMz3c="
|
||||
},
|
||||
"combined-stream": {
|
||||
"version": "1.0.8",
|
||||
"resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz",
|
||||
"integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==",
|
||||
"optional": true,
|
||||
"requires": {
|
||||
"delayed-stream": "~1.0.0"
|
||||
}
|
||||
},
|
||||
"concat-map": {
|
||||
"version": "0.0.1",
|
||||
"resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz",
|
||||
"integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s="
|
||||
},
|
||||
"console-control-strings": {
|
||||
"version": "1.1.0",
|
||||
"resolved": "https://registry.npmjs.org/console-control-strings/-/console-control-strings-1.1.0.tgz",
|
||||
"integrity": "sha1-PXz0Rk22RG6mRL9LOVB/mFEAjo4="
|
||||
},
|
||||
"core-util-is": {
|
||||
"version": "1.0.3",
|
||||
"resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.3.tgz",
|
||||
"integrity": "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ=="
|
||||
},
|
||||
"dashdash": {
|
||||
"version": "1.14.1",
|
||||
"resolved": "https://registry.npmjs.org/dashdash/-/dashdash-1.14.1.tgz",
|
||||
"integrity": "sha1-hTz6D3y+L+1d4gMmuN1YEDX24vA=",
|
||||
"optional": true,
|
||||
"requires": {
|
||||
"assert-plus": "^1.0.0"
|
||||
}
|
||||
},
|
||||
"debug": {
|
||||
"version": "3.2.7",
|
||||
"resolved": "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz",
|
||||
"integrity": "sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==",
|
||||
"requires": {
|
||||
"ms": "^2.1.1"
|
||||
}
|
||||
},
|
||||
"deep-extend": {
|
||||
"version": "0.6.0",
|
||||
"resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz",
|
||||
"integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA=="
|
||||
},
|
||||
"delayed-stream": {
|
||||
"version": "1.0.0",
|
||||
"resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz",
|
||||
"integrity": "sha1-3zrhmayt+31ECqrgsp4icrJOxhk=",
|
||||
"optional": true
|
||||
},
|
||||
"delegates": {
|
||||
"version": "1.0.0",
|
||||
"resolved": "https://registry.npmjs.org/delegates/-/delegates-1.0.0.tgz",
|
||||
"integrity": "sha1-hMbhWbgZBP3KWaDvRM2HDTElD5o="
|
||||
},
|
||||
"detect-libc": {
|
||||
"version": "1.0.3",
|
||||
"resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-1.0.3.tgz",
|
||||
"integrity": "sha1-+hN8S9aY7fVc1c0CrFWfkaTEups="
|
||||
},
|
||||
"ecc-jsbn": {
|
||||
"version": "0.1.2",
|
||||
"resolved": "https://registry.npmjs.org/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz",
|
||||
"integrity": "sha1-OoOpBOVDUyh4dMVkt1SThoSamMk=",
|
||||
"optional": true,
|
||||
"requires": {
|
||||
"jsbn": "~0.1.0",
|
||||
"safer-buffer": "^2.1.0"
|
||||
}
|
||||
},
|
||||
"extend": {
|
||||
"version": "3.0.2",
|
||||
"resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz",
|
||||
"integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==",
|
||||
"optional": true
|
||||
},
|
||||
"extsprintf": {
|
||||
"version": "1.3.0",
|
||||
"resolved": "https://registry.npmjs.org/extsprintf/-/extsprintf-1.3.0.tgz",
|
||||
"integrity": "sha1-lpGEQOMEGnpBT4xS48V06zw+HgU=",
|
||||
"optional": true
|
||||
},
|
||||
"fast-deep-equal": {
|
||||
"version": "3.1.3",
|
||||
"resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz",
|
||||
"integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==",
|
||||
"optional": true
|
||||
},
|
||||
"fast-json-stable-stringify": {
|
||||
"version": "2.1.0",
|
||||
"resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz",
|
||||
"integrity": "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==",
|
||||
"optional": true
|
||||
},
|
||||
"forever-agent": {
|
||||
"version": "0.6.1",
|
||||
"resolved": "https://registry.npmjs.org/forever-agent/-/forever-agent-0.6.1.tgz",
|
||||
"integrity": "sha1-+8cfDEGt6zf5bFd60e1C2P2sypE=",
|
||||
"optional": true
|
||||
},
|
||||
"form-data": {
|
||||
"version": "2.3.3",
|
||||
"resolved": "https://registry.npmjs.org/form-data/-/form-data-2.3.3.tgz",
|
||||
"integrity": "sha512-1lLKB2Mu3aGP1Q/2eCOx0fNbRMe7XdwktwOruhfqqd0rIJWwN4Dh+E3hrPSlDCXnSR7UtZ1N38rVXm+6+MEhJQ==",
|
||||
"optional": true,
|
||||
"requires": {
|
||||
"asynckit": "^0.4.0",
|
||||
"combined-stream": "^1.0.6",
|
||||
"mime-types": "^2.1.12"
|
||||
}
|
||||
},
|
||||
"fs-minipass": {
|
||||
"version": "1.2.7",
|
||||
"resolved": "https://registry.npmjs.org/fs-minipass/-/fs-minipass-1.2.7.tgz",
|
||||
"integrity": "sha512-GWSSJGFy4e9GUeCcbIkED+bgAoFyj7XF1mV8rma3QW4NIqX9Kyx79N/PF61H5udOV3aY1IaMLs6pGbH71nlCTA==",
|
||||
"requires": {
|
||||
"minipass": "^2.6.0"
|
||||
}
|
||||
},
|
||||
"fs.realpath": {
|
||||
"version": "1.0.0",
|
||||
"resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz",
|
||||
"integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8="
|
||||
},
|
||||
"fstream": {
|
||||
"version": "1.0.12",
|
||||
"resolved": "https://registry.npmjs.org/fstream/-/fstream-1.0.12.tgz",
|
||||
"integrity": "sha512-WvJ193OHa0GHPEL+AycEJgxvBEwyfRkN1vhjca23OaPVMCaLCXTd5qAu82AjTcgP1UJmytkOKb63Ypde7raDIg==",
|
||||
"optional": true,
|
||||
"requires": {
|
||||
"graceful-fs": "^4.1.2",
|
||||
"inherits": "~2.0.0",
|
||||
"mkdirp": ">=0.5 0",
|
||||
"rimraf": "2"
|
||||
}
|
||||
},
|
||||
"gauge": {
|
||||
"version": "2.7.4",
|
||||
"resolved": "https://registry.npmjs.org/gauge/-/gauge-2.7.4.tgz",
|
||||
"integrity": "sha1-LANAXHU4w51+s3sxcCLjJfsBi/c=",
|
||||
"requires": {
|
||||
"aproba": "^1.0.3",
|
||||
"console-control-strings": "^1.0.0",
|
||||
"has-unicode": "^2.0.0",
|
||||
"object-assign": "^4.1.0",
|
||||
"signal-exit": "^3.0.0",
|
||||
"string-width": "^1.0.1",
|
||||
"strip-ansi": "^3.0.1",
|
||||
"wide-align": "^1.1.0"
|
||||
}
|
||||
},
|
||||
"getpass": {
|
||||
"version": "0.1.7",
|
||||
"resolved": "https://registry.npmjs.org/getpass/-/getpass-0.1.7.tgz",
|
||||
"integrity": "sha1-Xv+OPmhNVprkyysSgmBOi6YhSfo=",
|
||||
"optional": true,
|
||||
"requires": {
|
||||
"assert-plus": "^1.0.0"
|
||||
}
|
||||
},
|
||||
"glob": {
|
||||
"version": "7.2.0",
|
||||
"resolved": "https://registry.npmjs.org/glob/-/glob-7.2.0.tgz",
|
||||
"integrity": "sha512-lmLf6gtyrPq8tTjSmrO94wBeQbFR3HbLHbuyD69wuyQkImp2hWqMGB47OX65FBkPffO641IP9jWa1z4ivqG26Q==",
|
||||
"requires": {
|
||||
"fs.realpath": "^1.0.0",
|
||||
"inflight": "^1.0.4",
|
||||
"inherits": "2",
|
||||
"minimatch": "^3.0.4",
|
||||
"once": "^1.3.0",
|
||||
"path-is-absolute": "^1.0.0"
|
||||
}
|
||||
},
|
||||
"graceful-fs": {
|
||||
"version": "4.2.8",
|
||||
"resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.8.tgz",
|
||||
"integrity": "sha512-qkIilPUYcNhJpd33n0GBXTB1MMPp14TxEsEs0pTrsSVucApsYzW5V+Q8Qxhik6KU3evy+qkAAowTByymK0avdg==",
|
||||
"optional": true
|
||||
},
|
||||
"har-schema": {
|
||||
"version": "2.0.0",
|
||||
"resolved": "https://registry.npmjs.org/har-schema/-/har-schema-2.0.0.tgz",
|
||||
"integrity": "sha1-qUwiJOvKwEeCoNkDVSHyRzW37JI=",
|
||||
"optional": true
|
||||
},
|
||||
"har-validator": {
|
||||
"version": "5.1.5",
|
||||
"resolved": "https://registry.npmjs.org/har-validator/-/har-validator-5.1.5.tgz",
|
||||
"integrity": "sha512-nmT2T0lljbxdQZfspsno9hgrG3Uir6Ks5afism62poxqBM6sDnMEuPmzTq8XN0OEwqKLLdh1jQI3qyE66Nzb3w==",
|
||||
"optional": true,
|
||||
"requires": {
|
||||
"ajv": "^6.12.3",
|
||||
"har-schema": "^2.0.0"
|
||||
}
|
||||
},
|
||||
"has-unicode": {
|
||||
"version": "2.0.1",
|
||||
"resolved": "https://registry.npmjs.org/has-unicode/-/has-unicode-2.0.1.tgz",
|
||||
"integrity": "sha1-4Ob+aijPUROIVeCG0Wkedx3iqLk="
|
||||
},
|
||||
"http-signature": {
|
||||
"version": "1.2.0",
|
||||
"resolved": "https://registry.npmjs.org/http-signature/-/http-signature-1.2.0.tgz",
|
||||
"integrity": "sha1-muzZJRFHcvPZW2WmCruPfBj7rOE=",
|
||||
"optional": true,
|
||||
"requires": {
|
||||
"assert-plus": "^1.0.0",
|
||||
"jsprim": "^1.2.2",
|
||||
"sshpk": "^1.7.0"
|
||||
}
|
||||
},
|
||||
"iconv-lite": {
|
||||
"version": "0.4.24",
|
||||
"resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz",
|
||||
"integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==",
|
||||
"requires": {
|
||||
"safer-buffer": ">= 2.1.2 < 3"
|
||||
}
|
||||
},
|
||||
"ignore-walk": {
|
||||
"version": "3.0.4",
|
||||
"resolved": "https://registry.npmjs.org/ignore-walk/-/ignore-walk-3.0.4.tgz",
|
||||
"integrity": "sha512-PY6Ii8o1jMRA1z4F2hRkH/xN59ox43DavKvD3oDpfurRlOJyAHpifIwpbdv1n4jt4ov0jSpw3kQ4GhJnpBL6WQ==",
|
||||
"requires": {
|
||||
"minimatch": "^3.0.4"
|
||||
}
|
||||
},
|
||||
"inflight": {
|
||||
"version": "1.0.6",
|
||||
"resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz",
|
||||
"integrity": "sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=",
|
||||
"requires": {
|
||||
"once": "^1.3.0",
|
||||
"wrappy": "1"
|
||||
}
|
||||
},
|
||||
"inherits": {
|
||||
"version": "2.0.4",
|
||||
"resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz",
|
||||
"integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ=="
|
||||
},
|
||||
"ini": {
|
||||
"version": "1.3.8",
|
||||
"resolved": "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz",
|
||||
"integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew=="
|
||||
},
|
||||
"is-fullwidth-code-point": {
|
||||
"version": "1.0.0",
|
||||
"resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-1.0.0.tgz",
|
||||
"integrity": "sha1-754xOG8DGn8NZDr4L95QxFfvAMs=",
|
||||
"requires": {
|
||||
"number-is-nan": "^1.0.0"
|
||||
}
|
||||
},
|
||||
"is-typedarray": {
|
||||
"version": "1.0.0",
|
||||
"resolved": "https://registry.npmjs.org/is-typedarray/-/is-typedarray-1.0.0.tgz",
|
||||
"integrity": "sha1-5HnICFjfDBsR3dppQPlgEfzaSpo=",
|
||||
"optional": true
|
||||
},
|
||||
"isarray": {
|
||||
"version": "1.0.0",
|
||||
"resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz",
|
||||
"integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE="
|
||||
},
|
||||
"isexe": {
|
||||
"version": "2.0.0",
|
||||
"resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz",
|
||||
"integrity": "sha1-6PvzdNxVb/iUehDcsFctYz8s+hA=",
|
||||
"optional": true
|
||||
},
|
||||
"isstream": {
|
||||
"version": "0.1.2",
|
||||
"resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz",
|
||||
"integrity": "sha1-R+Y/evVa+m+S4VAOaQ64uFKcCZo=",
|
||||
"optional": true
|
||||
},
|
||||
"jsbn": {
|
||||
"version": "0.1.1",
|
||||
"resolved": "https://registry.npmjs.org/jsbn/-/jsbn-0.1.1.tgz",
|
||||
"integrity": "sha1-peZUwuWi3rXyAdls77yoDA7y9RM=",
|
||||
"optional": true
|
||||
},
|
||||
"json-schema": {
|
||||
"version": "0.2.3",
|
||||
"resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.2.3.tgz",
|
||||
"integrity": "sha1-tIDIkuWaLwWVTOcnvT8qTogvnhM=",
|
||||
"optional": true
|
||||
},
|
||||
"json-schema-traverse": {
|
||||
"version": "0.4.1",
|
||||
"resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz",
|
||||
"integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==",
|
||||
"optional": true
|
||||
},
|
||||
"json-stringify-safe": {
|
||||
"version": "5.0.1",
|
||||
"resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz",
|
||||
"integrity": "sha1-Epai1Y/UXxmg9s4B1lcB4sc1tus=",
|
||||
"optional": true
|
||||
},
|
||||
"jsprim": {
|
||||
"version": "1.4.1",
|
||||
"resolved": "https://registry.npmjs.org/jsprim/-/jsprim-1.4.1.tgz",
|
||||
"integrity": "sha1-MT5mvB5cwG5Di8G3SZwuXFastqI=",
|
||||
"optional": true,
|
||||
"requires": {
|
||||
"assert-plus": "1.0.0",
|
||||
"extsprintf": "1.3.0",
|
||||
"json-schema": "0.2.3",
|
||||
"verror": "1.10.0"
|
||||
}
|
||||
},
|
||||
"mime-db": {
|
||||
"version": "1.49.0",
|
||||
"resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.49.0.tgz",
|
||||
"integrity": "sha512-CIc8j9URtOVApSFCQIF+VBkX1RwXp/oMMOrqdyXSBXq5RWNEsRfyj1kiRnQgmNXmHxPoFIxOroKA3zcU9P+nAA==",
|
||||
"optional": true
|
||||
},
|
||||
"mime-types": {
|
||||
"version": "2.1.32",
|
||||
"resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.32.tgz",
|
||||
"integrity": "sha512-hJGaVS4G4c9TSMYh2n6SQAGrC4RnfU+daP8G7cSCmaqNjiOoUY0VHCMS42pxnQmVF1GWwFhbHWn3RIxCqTmZ9A==",
|
||||
"optional": true,
|
||||
"requires": {
|
||||
"mime-db": "1.49.0"
|
||||
}
|
||||
},
|
||||
"minimatch": {
|
||||
"version": "3.0.4",
|
||||
"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz",
|
||||
"integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==",
|
||||
"requires": {
|
||||
"brace-expansion": "^1.1.7"
|
||||
}
|
||||
},
|
||||
"minimist": {
|
||||
"version": "1.2.5",
|
||||
"resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.5.tgz",
|
||||
"integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw=="
|
||||
},
|
||||
"minipass": {
|
||||
"version": "2.9.0",
|
||||
"resolved": "https://registry.npmjs.org/minipass/-/minipass-2.9.0.tgz",
|
||||
"integrity": "sha512-wxfUjg9WebH+CUDX/CdbRlh5SmfZiy/hpkxaRI16Y9W56Pa75sWgd/rvFilSgrauD9NyFymP/+JFV3KwzIsJeg==",
|
||||
"requires": {
|
||||
"safe-buffer": "^5.1.2",
|
||||
"yallist": "^3.0.0"
|
||||
}
|
||||
},
|
||||
"minizlib": {
|
||||
"version": "1.3.3",
|
||||
"resolved": "https://registry.npmjs.org/minizlib/-/minizlib-1.3.3.tgz",
|
||||
"integrity": "sha512-6ZYMOEnmVsdCeTJVE0W9ZD+pVnE8h9Hma/iOwwRDsdQoePpoX56/8B6z3P9VNwppJuBKNRuFDRNRqRWexT9G9Q==",
|
||||
"requires": {
|
||||
"minipass": "^2.9.0"
|
||||
}
|
||||
},
|
||||
"mkdirp": {
|
||||
"version": "0.5.5",
|
||||
"resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.5.tgz",
|
||||
"integrity": "sha512-NKmAlESf6jMGym1++R0Ra7wvhV+wFW63FaSOFPwRahvea0gMUcGUhVeAg/0BC0wiv9ih5NYPB1Wn1UEI1/L+xQ==",
|
||||
"requires": {
|
||||
"minimist": "^1.2.5"
|
||||
}
|
||||
},
|
||||
"ms": {
|
||||
"version": "2.1.3",
|
||||
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",
|
||||
"integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="
|
||||
},
|
||||
"needle": {
|
||||
"version": "2.9.1",
|
||||
"resolved": "https://registry.npmjs.org/needle/-/needle-2.9.1.tgz",
|
||||
"integrity": "sha512-6R9fqJ5Zcmf+uYaFgdIHmLwNldn5HbK8L5ybn7Uz+ylX/rnOsSp1AHcvQSrCaFN+qNM1wpymHqD7mVasEOlHGQ==",
|
||||
"requires": {
|
||||
"debug": "^3.2.6",
|
||||
"iconv-lite": "^0.4.4",
|
||||
"sax": "^1.2.4"
|
||||
}
|
||||
},
|
||||
"node-addon-api": {
|
||||
"version": "3.2.1",
|
||||
"resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-3.2.1.tgz",
|
||||
"integrity": "sha512-mmcei9JghVNDYydghQmeDX8KoAm0FAiYyIcUt/N4nhyAipB17pllZQDOJD2fotxABnt4Mdz+dKTO7eftLg4d0A=="
|
||||
},
|
||||
"node-gyp": {
|
||||
"version": "3.8.0",
|
||||
"resolved": "https://registry.npmjs.org/node-gyp/-/node-gyp-3.8.0.tgz",
|
||||
"integrity": "sha512-3g8lYefrRRzvGeSowdJKAKyks8oUpLEd/DyPV4eMhVlhJ0aNaZqIrNUIPuEWWTAoPqyFkfGrM67MC69baqn6vA==",
|
||||
"optional": true,
|
||||
"requires": {
|
||||
"fstream": "^1.0.0",
|
||||
"glob": "^7.0.3",
|
||||
"graceful-fs": "^4.1.2",
|
||||
"mkdirp": "^0.5.0",
|
||||
"nopt": "2 || 3",
|
||||
"npmlog": "0 || 1 || 2 || 3 || 4",
|
||||
"osenv": "0",
|
||||
"request": "^2.87.0",
|
||||
"rimraf": "2",
|
||||
"semver": "~5.3.0",
|
||||
"tar": "^2.0.0",
|
||||
"which": "1"
|
||||
}
|
||||
},
|
||||
"node-pre-gyp": {
|
||||
"version": "0.11.0",
|
||||
"resolved": "https://registry.npmjs.org/node-pre-gyp/-/node-pre-gyp-0.11.0.tgz",
|
||||
"integrity": "sha512-TwWAOZb0j7e9eGaf9esRx3ZcLaE5tQ2lvYy1pb5IAaG1a2e2Kv5Lms1Y4hpj+ciXJRofIxxlt5haeQ/2ANeE0Q==",
|
||||
"requires": {
|
||||
"detect-libc": "^1.0.2",
|
||||
"mkdirp": "^0.5.1",
|
||||
"needle": "^2.2.1",
|
||||
"nopt": "^4.0.1",
|
||||
"npm-packlist": "^1.1.6",
|
||||
"npmlog": "^4.0.2",
|
||||
"rc": "^1.2.7",
|
||||
"rimraf": "^2.6.1",
|
||||
"semver": "^5.3.0",
|
||||
"tar": "^4"
|
||||
},
|
||||
"dependencies": {
|
||||
"nopt": {
|
||||
"version": "4.0.3",
|
||||
"resolved": "https://registry.npmjs.org/nopt/-/nopt-4.0.3.tgz",
|
||||
"integrity": "sha512-CvaGwVMztSMJLOeXPrez7fyfObdZqNUK1cPAEzLHrTybIua9pMdmmPR5YwtfNftIOMv3DPUhFaxsZMNTQO20Kg==",
|
||||
"requires": {
|
||||
"abbrev": "1",
|
||||
"osenv": "^0.1.4"
|
||||
}
|
||||
},
|
||||
"safe-buffer": {
|
||||
"version": "5.2.1",
|
||||
"resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz",
|
||||
"integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ=="
|
||||
},
|
||||
"tar": {
|
||||
"version": "4.4.19",
|
||||
"resolved": "https://registry.npmjs.org/tar/-/tar-4.4.19.tgz",
|
||||
"integrity": "sha512-a20gEsvHnWe0ygBY8JbxoM4w3SJdhc7ZAuxkLqh+nvNQN2IOt0B5lLgM490X5Hl8FF0dl0tOf2ewFYAlIFgzVA==",
|
||||
"requires": {
|
||||
"chownr": "^1.1.4",
|
||||
"fs-minipass": "^1.2.7",
|
||||
"minipass": "^2.9.0",
|
||||
"minizlib": "^1.3.3",
|
||||
"mkdirp": "^0.5.5",
|
||||
"safe-buffer": "^5.2.1",
|
||||
"yallist": "^3.1.1"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"nopt": {
|
||||
"version": "3.0.6",
|
||||
"resolved": "https://registry.npmjs.org/nopt/-/nopt-3.0.6.tgz",
|
||||
"integrity": "sha1-xkZdvwirzU2zWTF/eaxopkayj/k=",
|
||||
"optional": true,
|
||||
"requires": {
|
||||
"abbrev": "1"
|
||||
}
|
||||
},
|
||||
"npm-bundled": {
|
||||
"version": "1.1.2",
|
||||
"resolved": "https://registry.npmjs.org/npm-bundled/-/npm-bundled-1.1.2.tgz",
|
||||
"integrity": "sha512-x5DHup0SuyQcmL3s7Rx/YQ8sbw/Hzg0rj48eN0dV7hf5cmQq5PXIeioroH3raV1QC1yh3uTYuMThvEQF3iKgGQ==",
|
||||
"requires": {
|
||||
"npm-normalize-package-bin": "^1.0.1"
|
||||
}
|
||||
},
|
||||
"npm-normalize-package-bin": {
|
||||
"version": "1.0.1",
|
||||
"resolved": "https://registry.npmjs.org/npm-normalize-package-bin/-/npm-normalize-package-bin-1.0.1.tgz",
|
||||
"integrity": "sha512-EPfafl6JL5/rU+ot6P3gRSCpPDW5VmIzX959Ob1+ySFUuuYHWHekXpwdUZcKP5C+DS4GEtdJluwBjnsNDl+fSA=="
|
||||
},
|
||||
"npm-packlist": {
|
||||
"version": "1.4.8",
|
||||
"resolved": "https://registry.npmjs.org/npm-packlist/-/npm-packlist-1.4.8.tgz",
|
||||
"integrity": "sha512-5+AZgwru5IevF5ZdnFglB5wNlHG1AOOuw28WhUq8/8emhBmLv6jX5by4WJCh7lW0uSYZYS6DXqIsyZVIXRZU9A==",
|
||||
"requires": {
|
||||
"ignore-walk": "^3.0.1",
|
||||
"npm-bundled": "^1.0.1",
|
||||
"npm-normalize-package-bin": "^1.0.1"
|
||||
}
|
||||
},
|
||||
"npmlog": {
|
||||
"version": "4.1.2",
|
||||
"resolved": "https://registry.npmjs.org/npmlog/-/npmlog-4.1.2.tgz",
|
||||
"integrity": "sha512-2uUqazuKlTaSI/dC8AzicUck7+IrEaOnN/e0jd3Xtt1KcGpwx30v50mL7oPyr/h9bL3E4aZccVwpwP+5W9Vjkg==",
|
||||
"requires": {
|
||||
"are-we-there-yet": "~1.1.2",
|
||||
"console-control-strings": "~1.1.0",
|
||||
"gauge": "~2.7.3",
|
||||
"set-blocking": "~2.0.0"
|
||||
}
|
||||
},
|
||||
"number-is-nan": {
|
||||
"version": "1.0.1",
|
||||
"resolved": "https://registry.npmjs.org/number-is-nan/-/number-is-nan-1.0.1.tgz",
|
||||
"integrity": "sha1-CXtgK1NCKlIsGvuHkDGDNpQaAR0="
|
||||
},
|
||||
"oauth-sign": {
|
||||
"version": "0.9.0",
|
||||
"resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.9.0.tgz",
|
||||
"integrity": "sha512-fexhUFFPTGV8ybAtSIGbV6gOkSv8UtRbDBnAyLQw4QPKkgNlsH2ByPGtMUqdWkos6YCRmAqViwgZrJc/mRDzZQ==",
|
||||
"optional": true
|
||||
},
|
||||
"object-assign": {
|
||||
"version": "4.1.1",
|
||||
"resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz",
|
||||
"integrity": "sha1-IQmtx5ZYh8/AXLvUQsrIv7s2CGM="
|
||||
},
|
||||
"once": {
|
||||
"version": "1.4.0",
|
||||
"resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz",
|
||||
"integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=",
|
||||
"requires": {
|
||||
"wrappy": "1"
|
||||
}
|
||||
},
|
||||
"os-homedir": {
|
||||
"version": "1.0.2",
|
||||
"resolved": "https://registry.npmjs.org/os-homedir/-/os-homedir-1.0.2.tgz",
|
||||
"integrity": "sha1-/7xJiDNuDoM94MFox+8VISGqf7M="
|
||||
},
|
||||
"os-tmpdir": {
|
||||
"version": "1.0.2",
|
||||
"resolved": "https://registry.npmjs.org/os-tmpdir/-/os-tmpdir-1.0.2.tgz",
|
||||
"integrity": "sha1-u+Z0BseaqFxc/sdm/lc0VV36EnQ="
|
||||
},
|
||||
"osenv": {
|
||||
"version": "0.1.5",
|
||||
"resolved": "https://registry.npmjs.org/osenv/-/osenv-0.1.5.tgz",
|
||||
"integrity": "sha512-0CWcCECdMVc2Rw3U5w9ZjqX6ga6ubk1xDVKxtBQPK7wis/0F2r9T6k4ydGYhecl7YUBxBVxhL5oisPsNxAPe2g==",
|
||||
"requires": {
|
||||
"os-homedir": "^1.0.0",
|
||||
"os-tmpdir": "^1.0.0"
|
||||
}
|
||||
},
|
||||
"path-is-absolute": {
|
||||
"version": "1.0.1",
|
||||
"resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz",
|
||||
"integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18="
|
||||
},
|
||||
"performance-now": {
|
||||
"version": "2.1.0",
|
||||
"resolved": "https://registry.npmjs.org/performance-now/-/performance-now-2.1.0.tgz",
|
||||
"integrity": "sha1-Ywn04OX6kT7BxpMHrjZLSzd8nns=",
|
||||
"optional": true
|
||||
},
|
||||
"process-nextick-args": {
|
||||
"version": "2.0.1",
|
||||
"resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz",
|
||||
"integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag=="
|
||||
},
|
||||
"psl": {
|
||||
"version": "1.8.0",
|
||||
"resolved": "https://registry.npmjs.org/psl/-/psl-1.8.0.tgz",
|
||||
"integrity": "sha512-RIdOzyoavK+hA18OGGWDqUTsCLhtA7IcZ/6NCs4fFJaHBDab+pDDmDIByWFRQJq2Cd7r1OoQxBGKOaztq+hjIQ==",
|
||||
"optional": true
|
||||
},
|
||||
"punycode": {
|
||||
"version": "2.1.1",
|
||||
"resolved": "https://registry.npmjs.org/punycode/-/punycode-2.1.1.tgz",
|
||||
"integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==",
|
||||
"optional": true
|
||||
},
|
||||
"qs": {
|
||||
"version": "6.5.2",
|
||||
"resolved": "https://registry.npmjs.org/qs/-/qs-6.5.2.tgz",
|
||||
"integrity": "sha512-N5ZAX4/LxJmF+7wN74pUD6qAh9/wnvdQcjq9TZjevvXzSUo7bfmw91saqMjzGS2xq91/odN2dW/WOl7qQHNDGA==",
|
||||
"optional": true
|
||||
},
|
||||
"rc": {
|
||||
"version": "1.2.8",
|
||||
"resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz",
|
||||
"integrity": "sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==",
|
||||
"requires": {
|
||||
"deep-extend": "^0.6.0",
|
||||
"ini": "~1.3.0",
|
||||
"minimist": "^1.2.0",
|
||||
"strip-json-comments": "~2.0.1"
|
||||
}
|
||||
},
|
||||
"readable-stream": {
|
||||
"version": "2.3.7",
|
||||
"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz",
|
||||
"integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==",
|
||||
"requires": {
|
||||
"core-util-is": "~1.0.0",
|
||||
"inherits": "~2.0.3",
|
||||
"isarray": "~1.0.0",
|
||||
"process-nextick-args": "~2.0.0",
|
||||
"safe-buffer": "~5.1.1",
|
||||
"string_decoder": "~1.1.1",
|
||||
"util-deprecate": "~1.0.1"
|
||||
}
|
||||
},
|
||||
"request": {
|
||||
"version": "2.88.2",
|
||||
"resolved": "https://registry.npmjs.org/request/-/request-2.88.2.tgz",
|
||||
"integrity": "sha512-MsvtOrfG9ZcrOwAW+Qi+F6HbD0CWXEh9ou77uOb7FM2WPhwT7smM833PzanhJLsgXjN89Ir6V2PczXNnMpwKhw==",
|
||||
"optional": true,
|
||||
"requires": {
|
||||
"aws-sign2": "~0.7.0",
|
||||
"aws4": "^1.8.0",
|
||||
"caseless": "~0.12.0",
|
||||
"combined-stream": "~1.0.6",
|
||||
"extend": "~3.0.2",
|
||||
"forever-agent": "~0.6.1",
|
||||
"form-data": "~2.3.2",
|
||||
"har-validator": "~5.1.3",
|
||||
"http-signature": "~1.2.0",
|
||||
"is-typedarray": "~1.0.0",
|
||||
"isstream": "~0.1.2",
|
||||
"json-stringify-safe": "~5.0.1",
|
||||
"mime-types": "~2.1.19",
|
||||
"oauth-sign": "~0.9.0",
|
||||
"performance-now": "^2.1.0",
|
||||
"qs": "~6.5.2",
|
||||
"safe-buffer": "^5.1.2",
|
||||
"tough-cookie": "~2.5.0",
|
||||
"tunnel-agent": "^0.6.0",
|
||||
"uuid": "^3.3.2"
|
||||
}
|
||||
},
|
||||
"rimraf": {
|
||||
"version": "2.7.1",
|
||||
"resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.7.1.tgz",
|
||||
"integrity": "sha512-uWjbaKIK3T1OSVptzX7Nl6PvQ3qAGtKEtVRjRuazjfL3Bx5eI409VZSqgND+4UNnmzLVdPj9FqFJNPqBZFve4w==",
|
||||
"requires": {
|
||||
"glob": "^7.1.3"
|
||||
}
|
||||
},
|
||||
"safe-buffer": {
|
||||
"version": "5.1.2",
|
||||
"resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz",
|
||||
"integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g=="
|
||||
},
|
||||
"safer-buffer": {
|
||||
"version": "2.1.2",
|
||||
"resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz",
|
||||
"integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg=="
|
||||
},
|
||||
"sax": {
|
||||
"version": "1.2.4",
|
||||
"resolved": "https://registry.npmjs.org/sax/-/sax-1.2.4.tgz",
|
||||
"integrity": "sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw=="
|
||||
},
|
||||
"semver": {
|
||||
"version": "5.3.0",
|
||||
"resolved": "https://registry.npmjs.org/semver/-/semver-5.3.0.tgz",
|
||||
"integrity": "sha1-myzl094C0XxgEq0yaqa00M9U+U8="
|
||||
},
|
||||
"set-blocking": {
|
||||
"version": "2.0.0",
|
||||
"resolved": "https://registry.npmjs.org/set-blocking/-/set-blocking-2.0.0.tgz",
|
||||
"integrity": "sha1-BF+XgtARrppoA93TgrJDkrPYkPc="
|
||||
},
|
||||
"signal-exit": {
|
||||
"version": "3.0.4",
|
||||
"resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.4.tgz",
|
||||
"integrity": "sha512-rqYhcAnZ6d/vTPGghdrw7iumdcbXpsk1b8IG/rz+VWV51DM0p7XCtMoJ3qhPLIbp3tvyt3pKRbaaEMZYpHto8Q=="
|
||||
},
|
||||
"sqlite": {
|
||||
"version": "4.0.23",
|
||||
"resolved": "https://registry.npmjs.org/sqlite/-/sqlite-4.0.23.tgz",
|
||||
"integrity": "sha512-dSdmSkrdIhUL7xP/fiEMfFuAo4dxb0afag3rK8T4Y9lYxE3g3fXT0J8H9qSFvmcKxnM0zEA8yvLbpdWQ8mom3g=="
|
||||
},
|
||||
"sqlite3": {
|
||||
"version": "5.0.2",
|
||||
"resolved": "https://registry.npmjs.org/sqlite3/-/sqlite3-5.0.2.tgz",
|
||||
"integrity": "sha512-1SdTNo+BVU211Xj1csWa8lV6KM0CtucDwRyA0VHl91wEH1Mgh7RxUpI4rVvG7OhHrzCSGaVyW5g8vKvlrk9DJA==",
|
||||
"requires": {
|
||||
"node-addon-api": "^3.0.0",
|
||||
"node-gyp": "3.x",
|
||||
"node-pre-gyp": "^0.11.0"
|
||||
}
|
||||
},
|
||||
"sshpk": {
|
||||
"version": "1.16.1",
|
||||
"resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.16.1.tgz",
|
||||
"integrity": "sha512-HXXqVUq7+pcKeLqqZj6mHFUMvXtOJt1uoUx09pFW6011inTMxqI8BA8PM95myrIyyKwdnzjdFjLiE6KBPVtJIg==",
|
||||
"optional": true,
|
||||
"requires": {
|
||||
"asn1": "~0.2.3",
|
||||
"assert-plus": "^1.0.0",
|
||||
"bcrypt-pbkdf": "^1.0.0",
|
||||
"dashdash": "^1.12.0",
|
||||
"ecc-jsbn": "~0.1.1",
|
||||
"getpass": "^0.1.1",
|
||||
"jsbn": "~0.1.0",
|
||||
"safer-buffer": "^2.0.2",
|
||||
"tweetnacl": "~0.14.0"
|
||||
}
|
||||
},
|
||||
"string-width": {
|
||||
"version": "1.0.2",
|
||||
"resolved": "https://registry.npmjs.org/string-width/-/string-width-1.0.2.tgz",
|
||||
"integrity": "sha1-EYvfW4zcUaKn5w0hHgfisLmxB9M=",
|
||||
"requires": {
|
||||
"code-point-at": "^1.0.0",
|
||||
"is-fullwidth-code-point": "^1.0.0",
|
||||
"strip-ansi": "^3.0.0"
|
||||
}
|
||||
},
|
||||
"string_decoder": {
|
||||
"version": "1.1.1",
|
||||
"resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz",
|
||||
"integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==",
|
||||
"requires": {
|
||||
"safe-buffer": "~5.1.0"
|
||||
}
|
||||
},
|
||||
"strip-ansi": {
|
||||
"version": "3.0.1",
|
||||
"resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz",
|
||||
"integrity": "sha1-ajhfuIU9lS1f8F0Oiq+UJ43GPc8=",
|
||||
"requires": {
|
||||
"ansi-regex": "^2.0.0"
|
||||
}
|
||||
},
|
||||
"strip-json-comments": {
|
||||
"version": "2.0.1",
|
||||
"resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz",
|
||||
"integrity": "sha1-PFMZQukIwml8DsNEhYwobHygpgo="
|
||||
},
|
||||
"tar": {
|
||||
"version": "2.2.2",
|
||||
"resolved": "https://registry.npmjs.org/tar/-/tar-2.2.2.tgz",
|
||||
"integrity": "sha512-FCEhQ/4rE1zYv9rYXJw/msRqsnmlje5jHP6huWeBZ704jUTy02c5AZyWujpMR1ax6mVw9NyJMfuK2CMDWVIfgA==",
|
||||
"optional": true,
|
||||
"requires": {
|
||||
"block-stream": "*",
|
||||
"fstream": "^1.0.12",
|
||||
"inherits": "2"
|
||||
}
|
||||
},
|
||||
"tough-cookie": {
|
||||
"version": "2.5.0",
|
||||
"resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.5.0.tgz",
|
||||
"integrity": "sha512-nlLsUzgm1kfLXSXfRZMc1KLAugd4hqJHDTvc2hDIwS3mZAfMEuMbc03SujMF+GEcpaX/qboeycw6iO8JwVv2+g==",
|
||||
"optional": true,
|
||||
"requires": {
|
||||
"psl": "^1.1.28",
|
||||
"punycode": "^2.1.1"
|
||||
}
|
||||
},
|
||||
"tunnel-agent": {
|
||||
"version": "0.6.0",
|
||||
"resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz",
|
||||
"integrity": "sha1-J6XeoGs2sEoKmWZ3SykIaPD8QP0=",
|
||||
"optional": true,
|
||||
"requires": {
|
||||
"safe-buffer": "^5.0.1"
|
||||
}
|
||||
},
|
||||
"tweetnacl": {
|
||||
"version": "0.14.5",
|
||||
"resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-0.14.5.tgz",
|
||||
"integrity": "sha1-WuaBd/GS1EViadEIr6k/+HQ/T2Q=",
|
||||
"optional": true
|
||||
},
|
||||
"uri-js": {
|
||||
"version": "4.4.1",
|
||||
"resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.4.1.tgz",
|
||||
"integrity": "sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==",
|
||||
"optional": true,
|
||||
"requires": {
|
||||
"punycode": "^2.1.0"
|
||||
}
|
||||
},
|
||||
"util-deprecate": {
|
||||
"version": "1.0.2",
|
||||
"resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz",
|
||||
"integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8="
|
||||
},
|
||||
"uuid": {
|
||||
"version": "3.4.0",
|
||||
"resolved": "https://registry.npmjs.org/uuid/-/uuid-3.4.0.tgz",
|
||||
"integrity": "sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==",
|
||||
"optional": true
|
||||
},
|
||||
"verror": {
|
||||
"version": "1.10.0",
|
||||
"resolved": "https://registry.npmjs.org/verror/-/verror-1.10.0.tgz",
|
||||
"integrity": "sha1-OhBcoXBTr1XW4nDB+CiGguGNpAA=",
|
||||
"optional": true,
|
||||
"requires": {
|
||||
"assert-plus": "^1.0.0",
|
||||
"core-util-is": "1.0.2",
|
||||
"extsprintf": "^1.2.0"
|
||||
},
|
||||
"dependencies": {
|
||||
"core-util-is": {
|
||||
"version": "1.0.2",
|
||||
"resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz",
|
||||
"integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=",
|
||||
"optional": true
|
||||
}
|
||||
}
|
||||
},
|
||||
"which": {
|
||||
"version": "1.3.1",
|
||||
"resolved": "https://registry.npmjs.org/which/-/which-1.3.1.tgz",
|
||||
"integrity": "sha512-HxJdYWq1MTIQbJ3nw0cqssHoTNU267KlrDuGZ1WYlxDStUtKUhOaJmh112/TZmHxxUfuJqPXSOm7tDyas0OSIQ==",
|
||||
"optional": true,
|
||||
"requires": {
|
||||
"isexe": "^2.0.0"
|
||||
}
|
||||
},
|
||||
"wide-align": {
|
||||
"version": "1.1.3",
|
||||
"resolved": "https://registry.npmjs.org/wide-align/-/wide-align-1.1.3.tgz",
|
||||
"integrity": "sha512-QGkOQc8XL6Bt5PwnsExKBPuMKBxnGxWWW3fU55Xt4feHozMUhdUMaBCk290qpm/wG5u/RSKzwdAC4i51YigihA==",
|
||||
"requires": {
|
||||
"string-width": "^1.0.2 || 2"
|
||||
}
|
||||
},
|
||||
"wrappy": {
|
||||
"version": "1.0.2",
|
||||
"resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz",
|
||||
"integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8="
|
||||
},
|
||||
"yallist": {
|
||||
"version": "3.1.1",
|
||||
"resolved": "https://registry.npmjs.org/yallist/-/yallist-3.1.1.tgz",
|
||||
"integrity": "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g=="
|
||||
}
|
||||
}
|
||||
}
|
||||
15
02-ingest-sqlite/package.json
Normal file
15
02-ingest-sqlite/package.json
Normal file
@ -0,0 +1,15 @@
|
||||
{
|
||||
"name": "02-ingest-sqlite",
|
||||
"version": "1.0.0",
|
||||
"description": "",
|
||||
"main": "01-injest-sqlite.js",
|
||||
"scripts": {
|
||||
"test": "echo \"Error: no test specified\" && exit 1"
|
||||
},
|
||||
"author": "",
|
||||
"license": "ISC",
|
||||
"dependencies": {
|
||||
"sqlite": "^4.0.23",
|
||||
"sqlite3": "^5.0.2"
|
||||
}
|
||||
}
|
||||
0
03-splitter-sqlite/input/.keep
Normal file
0
03-splitter-sqlite/input/.keep
Normal file
241
04-scraper-tabs/01-scrape-tabs.js
Normal file
241
04-scraper-tabs/01-scrape-tabs.js
Normal file
@ -0,0 +1,241 @@
|
||||
// Scrapes tabs from ultimate-guitar.com to complete the database
|
||||
|
||||
// node-fetch is an asshole that wants to be ESM-only so we have to do special stuff to import it easily
|
||||
const fetch = (...args) => import('node-fetch').then(({default: fetch}) => fetch(...args));
|
||||
const jsdom = require('jsdom');
|
||||
|
||||
const sqlite3 = require('sqlite3');
|
||||
const sqlite = require('sqlite');
|
||||
|
||||
const ConcurrentQueue = require('./concurrent-queue.js');
|
||||
|
||||
class ScrapeError extends Error {
|
||||
constructor(message, options, fileName, lineNumber) {
|
||||
super(...arguments);
|
||||
this.name = 'ScrapeError';
|
||||
}
|
||||
}
|
||||
|
||||
function estMSRemaining(startTime, ratioComplete) {
|
||||
return (1 - ratioComplete) * ((Date.now() - startTime) / ratioComplete);
|
||||
}
|
||||
|
||||
function formatRelative(msRelative) {
|
||||
if (msRelative < 1000) return `${(msRelative).toFixed(2)}ms`;
|
||||
else if (msRelative < 60 * 1000) return `${(msRelative / 1000).toFixed(2)}s`;
|
||||
else if (msRelative < 60 * 60 * 1000) return `${(msRelative / (60 * 1000)).toFixed(2)} mins`;
|
||||
else return `${(msRelative / (60 * 60 * 1000)).toFixed(2)} hours`;
|
||||
}
|
||||
|
||||
function sleep(ms) {
|
||||
return new Promise((resolve) => {
|
||||
setTimeout(resolve, ms);
|
||||
});
|
||||
}
|
||||
|
||||
// modified from background-script to use jsdom
|
||||
async function scrapeUGDataContent(url) {
|
||||
let page = null;
|
||||
try {
|
||||
page = await fetch(url);
|
||||
} catch (e) {
|
||||
throw new ScrapeError('Unable to fetch url', { cause: e });
|
||||
}
|
||||
|
||||
let text = null;
|
||||
try {
|
||||
text = await page.text();
|
||||
} catch (e) {
|
||||
throw new ScrapeError('Unable to decode page', { cause: e });
|
||||
}
|
||||
|
||||
let dom = null;
|
||||
try {
|
||||
dom = new jsdom.JSDOM(text);
|
||||
} catch (e) {
|
||||
throw new ScrapeError('Unable to parse document', { cause: e });
|
||||
}
|
||||
|
||||
if (!dom.window || !dom.window.document) {
|
||||
throw new ScrapeError('Unable to parse document');
|
||||
}
|
||||
|
||||
let document = dom.window.document;
|
||||
|
||||
const jsStore = document.querySelector('.js-store');
|
||||
if (jsStore == null) {
|
||||
throw new ScrapeError('Unable to find .js-store element')
|
||||
}
|
||||
|
||||
const contentJSON = jsStore.getAttribute('data-content');
|
||||
if (contentJSON == null) {
|
||||
throw new ScrapeError('Unable to find data-content attribute on .js-store');
|
||||
}
|
||||
|
||||
const content = JSON.parse(contentJSON);
|
||||
return content;
|
||||
}
|
||||
|
||||
function parseGeneralTab(ugDataContent) {
|
||||
const store = ugDataContent.store; if (store === null) throw new ScrapeError('Unable to get ugDataContent.store');
|
||||
const page = store.page; if (page === null) throw new ScrapeError('Unable to get ugDataContent.store.page');
|
||||
const data = page.data; if (data === null) throw new ScrapeError('Unable to get ugDataContent.store.page.data');
|
||||
const meta = data.tab; if (meta === null) throw new ScrapeError('Unable to get ugDataContent.store.page.data.tab');
|
||||
const tview = data.tab_view; if (tview === null) throw new ScrapeError('Unable to get ugDataContent.store.page.data.tab_view');
|
||||
const wktab = tview.wiki_tab; if (wktab === null) throw new ScrapeError('Unable to get ugDataContent.store.page.data.tab_view.wiki_tab');
|
||||
const text = wktab.content; if (text === null) throw new ScrapeError('Unable to get ugDataContent.store.page.data.tab_view.wiki_tab.content');
|
||||
return { meta, text };
|
||||
}
|
||||
|
||||
// Can only fetch Bass, Chords, Drums, Tab, and Ukulele type_name tabs
|
||||
async function fetchGeneralTab(url) {
|
||||
let ugDataContent = await scrapeUGDataContent(url);
|
||||
return parseGeneralTab(ugDataContent);
|
||||
}
|
||||
|
||||
(async () => {
|
||||
|
||||
const db = await sqlite.open({
|
||||
driver: sqlite3.Database,
|
||||
filename: './input/tabs.db'
|
||||
});
|
||||
|
||||
// Progress queries
|
||||
|
||||
let totalFetchable = (await db.get(`
|
||||
SELECT
|
||||
COUNT(*) AS c
|
||||
FROM
|
||||
tabs
|
||||
WHERE
|
||||
tab_url IS NOT NULL
|
||||
AND type_name IS NOT NULL
|
||||
AND (
|
||||
type_name='Bass'
|
||||
OR type_name='Chords'
|
||||
OR type_name='Drums'
|
||||
OR type_name='Tab'
|
||||
OR type_name='Ukulele'
|
||||
)
|
||||
`)).c;
|
||||
|
||||
console.log(`${totalFetchable} Total Fetchable Tabs`)
|
||||
|
||||
let completedFetchable = (await db.get(`
|
||||
SELECT
|
||||
COUNT(*) AS c
|
||||
FROM
|
||||
tabs
|
||||
WHERE
|
||||
tab_text IS NOT NULL
|
||||
AND tab_url IS NOT NULL
|
||||
AND type_name IS NOT NULL
|
||||
AND (
|
||||
type_name='Bass'
|
||||
OR type_name='Chords'
|
||||
OR type_name='Drums'
|
||||
OR type_name='Tab'
|
||||
OR type_name='Ukulele'
|
||||
)
|
||||
`)).c;
|
||||
|
||||
console.log(`${completedFetchable} (${(100 * completedFetchable / totalFetchable).toFixed(2)}%) Fetchable Tabs already completed`);
|
||||
|
||||
let stmtUpdateTab = await db.prepare(`
|
||||
UPDATE
|
||||
tabs
|
||||
SET
|
||||
user_id=?1
|
||||
, user_iq=?2
|
||||
, username=?3
|
||||
, tab_text=?4
|
||||
WHERE
|
||||
scrape_id=?5
|
||||
`);
|
||||
|
||||
// nigger :)
|
||||
|
||||
let remainingFetchable = totalFetchable - completedFetchable;
|
||||
let sessionCompleted = 0;
|
||||
let startTime = Date.now();
|
||||
let badUrls = new Set();
|
||||
while (true) {
|
||||
let queryStartTime = Date.now();
|
||||
let result = await db.all(`
|
||||
SELECT
|
||||
scrape_id
|
||||
, tab_url
|
||||
FROM
|
||||
tabs
|
||||
WHERE
|
||||
tab_text IS NULL
|
||||
AND tab_url IS NOT NULL
|
||||
AND type_name IS NOT NULL
|
||||
AND (
|
||||
type_name='Bass'
|
||||
OR type_name='Chords'
|
||||
OR type_name='Drums'
|
||||
OR type_name='Tab'
|
||||
OR type_name='Ukulele'
|
||||
)
|
||||
ORDER BY
|
||||
bucket
|
||||
LIMIT 300
|
||||
`);
|
||||
console.log(`SQLite Query took ${Date.now() - queryStartTime} ms`);
|
||||
// console.log('Sleeping for 10s');
|
||||
// await sleep(10000);
|
||||
|
||||
if (result.length === 0) break;
|
||||
|
||||
let batchCompleted = 0;
|
||||
let queue = new ConcurrentQueue(5);
|
||||
for (let tabInfo of result) {
|
||||
if (badUrls.has(tabInfo.tab_url)) continue;
|
||||
(async () => {
|
||||
try {
|
||||
await queue.push(async () => {
|
||||
let { meta, text } = await fetchGeneralTab(tabInfo.tab_url);
|
||||
let user_id = meta.user_id;
|
||||
let user_iq = meta.user_iq;
|
||||
let username = meta.username;
|
||||
let tab_text = text;
|
||||
await stmtUpdateTab.run([ user_id, user_iq, username, tab_text, tabInfo.scrape_id ]);
|
||||
batchCompleted += 1;
|
||||
if (batchCompleted % (Math.floor(result.length / 10)) === 0) {
|
||||
console.log(`batch completed: ${batchCompleted}/${result.length - badUrls.size}`);
|
||||
}
|
||||
});
|
||||
} catch (e) {
|
||||
console.error('Error fetching tab for ', tabInfo.tab_url, '. Error:', e.message);
|
||||
badUrls.add(tabInfo.tab_url);
|
||||
}
|
||||
})();
|
||||
await sleep(100);
|
||||
}
|
||||
await queue.waitForDrain();
|
||||
|
||||
sessionCompleted += batchCompleted;
|
||||
|
||||
let elapsed = formatRelative(Date.now() - startTime);
|
||||
let minsElapsed = (Date.now() - startTime) / (60 * 1000);
|
||||
let estimatedRemaining = formatRelative(estMSRemaining(startTime, sessionCompleted / remainingFetchable));
|
||||
let pctComplete = (100 * sessionCompleted / remainingFetchable);
|
||||
let pctPerMin = (pctComplete / minsElapsed);
|
||||
let tabsPerMin = (sessionCompleted / ((Date.now() - startTime) / (60 * 1000)));
|
||||
console.log('');
|
||||
console.log(`${sessionCompleted}/${remainingFetchable} tabs complete (${pctComplete.toFixed(2)}%)`);
|
||||
console.log(`${tabsPerMin.toFixed(2)} tabs/min (${pctPerMin.toFixed(5)} %/min)`);
|
||||
console.log(`${elapsed} elapsed (est. ${estimatedRemaining} remaining)`);
|
||||
console.log('');
|
||||
|
||||
if (batchCompleted / result.length < .5) {
|
||||
console.log('We got kicked off at ', new Date().toString());
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
await stmtUpdateTab.finalize();
|
||||
|
||||
await db.close();
|
||||
})();
|
||||
48
04-scraper-tabs/concurrent-queue.js
Normal file
48
04-scraper-tabs/concurrent-queue.js
Normal file
@ -0,0 +1,48 @@
|
||||
// Runs a limited number of promises at one time
|
||||
class ConcurrentQueue {
|
||||
constructor(consecutive) {
|
||||
this.consecutive = consecutive;
|
||||
this.queue = [];
|
||||
this.current = 0;
|
||||
this.drainListeners = [];
|
||||
}
|
||||
|
||||
_checkQueue() {
|
||||
if (this.current == 0 && this.queue.length == 0) {
|
||||
for (let drainListener of this.drainListeners) {
|
||||
drainListener();
|
||||
}
|
||||
this.drainListeners = [];
|
||||
}
|
||||
while (this.current < this.consecutive && this.queue.length > 0) {
|
||||
let taskData = this.queue.shift();
|
||||
this.current += 1;
|
||||
(async () => {
|
||||
try {
|
||||
taskData.resolve(await taskData.task());
|
||||
} catch (e) {
|
||||
taskData.reject(e);
|
||||
}
|
||||
this.current -= 1;
|
||||
this._checkQueue();
|
||||
})();
|
||||
}
|
||||
}
|
||||
|
||||
// returns a promise that can be awaited to get the resolution or rejection of the task's execution
|
||||
push(task) {
|
||||
return new Promise((resolve, reject) => {
|
||||
this.queue.push({ task, resolve, reject })
|
||||
this._checkQueue();
|
||||
});
|
||||
}
|
||||
|
||||
async waitForDrain() {
|
||||
return new Promise((resolve) => {
|
||||
this.drainListeners.push(resolve);
|
||||
this._checkQueue();
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = ConcurrentQueue;
|
||||
0
04-scraper-tabs/input/.keep
Normal file
0
04-scraper-tabs/input/.keep
Normal file
1350
04-scraper-tabs/package-lock.json
generated
Normal file
1350
04-scraper-tabs/package-lock.json
generated
Normal file
File diff suppressed because it is too large
Load Diff
17
04-scraper-tabs/package.json
Normal file
17
04-scraper-tabs/package.json
Normal file
@ -0,0 +1,17 @@
|
||||
{
|
||||
"name": "04-scraper-tabs",
|
||||
"version": "1.0.0",
|
||||
"description": "",
|
||||
"main": "01-scrape-tabs.js",
|
||||
"scripts": {
|
||||
"test": "echo \"Error: no test specified\" && exit 1"
|
||||
},
|
||||
"author": "",
|
||||
"license": "ISC",
|
||||
"dependencies": {
|
||||
"jsdom": "^17.0.0",
|
||||
"node-fetch": "^3.0.0",
|
||||
"sqlite": "^4.0.23",
|
||||
"sqlite3": "^5.0.2"
|
||||
}
|
||||
}
|
||||
0
05-merger-sqlite/input/.keep
Normal file
0
05-merger-sqlite/input/.keep
Normal file
126
06-output-generator/01-output-generator.js
Normal file
126
06-output-generator/01-output-generator.js
Normal file
@ -0,0 +1,126 @@
|
||||
const sqlite3 = require('sqlite3');
|
||||
const sqlite = require('sqlite');
|
||||
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
|
||||
const fsExtra = require('fs-extra');
|
||||
|
||||
function sanitizeFileName(name) {
|
||||
// Windows Version (created for Windows, most likely works cross-platform too given my research)
|
||||
// Allowed Characters: Extended Unicode Charset (1-255)
|
||||
// Illegal file names: CON, PRN, AUX, NUL, COM1, COM2, ..., COM9, LPT1, LPT2, ..., LPT9
|
||||
// Reserved Characters: <>:"/\|?*
|
||||
// Solution: Replace reserved characters with empty string (''), bad characters with '_', and append '_' to bad names
|
||||
|
||||
// Illegal File Names (Windows)
|
||||
if ([ 'CON', 'PRN', 'AUX', 'NUL',
|
||||
'COM1', 'COM2', 'COM3', 'COM4', 'COM5', 'COM6', 'COM7', 'COM8', 'COM9',
|
||||
'LPT1', 'LPT2', 'LPT3', 'LPT4', 'LPT5', 'LPT6', 'LPT7', 'LPT8', 'LPT9' ].indexOf(name) != -1) { // TODO: case insensitive?
|
||||
name += '_';
|
||||
}
|
||||
// Reserved Characters
|
||||
name = name.replace(/[<>:\"\/\\|?*]/g, '');
|
||||
// Allowed Characters
|
||||
return name.split('').map(c => c.charCodeAt(0) < 255 && c.charCodeAt(0) > 0 ? c : '_').join('');
|
||||
|
||||
// Much stricter whitelist version
|
||||
// replace bad characters with '_'
|
||||
//return name.split('').map(c => /[A-Za-z0-9-]/.exec(c) ? c : '_').join('');
|
||||
}
|
||||
|
||||
function getAvailableFileName(dir, name) {
|
||||
name = sanitizeFileName(name);
|
||||
let ext = path.extname(name);
|
||||
let baseName = path.basename(name, ext);
|
||||
let availableBaseName = baseName;
|
||||
let tries = 1;
|
||||
while (fs.existsSync(path.join(dir, availableBaseName + ext))) {
|
||||
availableBaseName = baseName + '-' + (++tries);
|
||||
}
|
||||
return availableBaseName + ext;
|
||||
}
|
||||
|
||||
(async () => {
|
||||
// Clear out old output directory
|
||||
await fsExtra.emptyDir('./output/');
|
||||
|
||||
const db = await sqlite.open({
|
||||
driver: sqlite3.Database,
|
||||
filename: './input/tabs-full.db'
|
||||
});
|
||||
|
||||
console.log('connected to db');
|
||||
|
||||
let total = (await db.get(`
|
||||
SELECT COUNT(*) AS c FROM tabs WHERE tab_text IS NOT NULL
|
||||
`)).c;
|
||||
|
||||
console.log(`${total} total tabs`);
|
||||
|
||||
let soFar = 0;
|
||||
const totalRows = await db.each(`
|
||||
SELECT
|
||||
scrape_id
|
||||
, id
|
||||
, song_id
|
||||
, song_name
|
||||
, artist_id
|
||||
, artist_name
|
||||
, version
|
||||
, version_description
|
||||
, votes
|
||||
, rating
|
||||
, date
|
||||
, tonality_name
|
||||
, verified
|
||||
, artist_url
|
||||
, tab_url
|
||||
, difficulty
|
||||
, tuning
|
||||
, type_name
|
||||
, user_id
|
||||
, user_iq
|
||||
, username
|
||||
, tab_text
|
||||
FROM tabs
|
||||
WHERE tab_text IS NOT NULL
|
||||
`, (err, row) => {
|
||||
if (err) throw err;
|
||||
|
||||
soFar += 1;
|
||||
let fileText =
|
||||
`${row.song_name} [${row.song_id}]: ${row.tab_url}
|
||||
By ${row.artist_name} [${row.artist_id}]: ${row.artist_url}
|
||||
Rating: ${row.rating}, Votes: ${row.votes}
|
||||
Date: ${row.date}
|
||||
Tonality: ${row.tonality_name}
|
||||
Difficulty: ${row.difficulty}
|
||||
Tuning: ${row.tuning}
|
||||
Type: ${row.type_name}
|
||||
Tab By: ${row.username} [${row.user_id}] (${row.user_iq} iq)
|
||||
Verified: ${row.verified}
|
||||
Version ${row.version}
|
||||
${row.version_description || ''}
|
||||
|
||||
${row.tab_text}
|
||||
`;
|
||||
let typeDir = path.join('output', row.type_name ?? 'null');
|
||||
if (!fs.existsSync(typeDir)) {
|
||||
fs.mkdirSync(typeDir);
|
||||
}
|
||||
|
||||
let fileDir = sanitizeFileName(row.artist_name + '-' + row.artist_id);
|
||||
if (!fs.existsSync(path.join(typeDir, fileDir))) {
|
||||
fs.mkdirSync(path.join(typeDir, fileDir));
|
||||
}
|
||||
|
||||
let fileName = getAvailableFileName(path.join(typeDir, fileDir), row.song_name + '.txt');
|
||||
fs.writeFileSync(path.join(typeDir, fileDir, fileName), fileText);
|
||||
|
||||
if (soFar % 100 == 0) {
|
||||
console.log(`Tab #${soFar}/${total} (${(100 * soFar / total).toFixed(2)}%): ${path.join(typeDir, fileDir, fileName)}`);
|
||||
}
|
||||
});
|
||||
|
||||
})();
|
||||
0
06-output-generator/input/.keep
Normal file
0
06-output-generator/input/.keep
Normal file
979
06-output-generator/package-lock.json
generated
Normal file
979
06-output-generator/package-lock.json
generated
Normal file
@ -0,0 +1,979 @@
|
||||
{
|
||||
"name": "06-output-generator",
|
||||
"version": "1.0.0",
|
||||
"lockfileVersion": 1,
|
||||
"requires": true,
|
||||
"dependencies": {
|
||||
"abbrev": {
|
||||
"version": "1.1.1",
|
||||
"resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz",
|
||||
"integrity": "sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q=="
|
||||
},
|
||||
"ajv": {
|
||||
"version": "6.12.6",
|
||||
"resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz",
|
||||
"integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==",
|
||||
"optional": true,
|
||||
"requires": {
|
||||
"fast-deep-equal": "^3.1.1",
|
||||
"fast-json-stable-stringify": "^2.0.0",
|
||||
"json-schema-traverse": "^0.4.1",
|
||||
"uri-js": "^4.2.2"
|
||||
}
|
||||
},
|
||||
"ansi-regex": {
|
||||
"version": "2.1.1",
|
||||
"resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz",
|
||||
"integrity": "sha1-w7M6te42DYbg5ijwRorn7yfWVN8="
|
||||
},
|
||||
"aproba": {
|
||||
"version": "1.2.0",
|
||||
"resolved": "https://registry.npmjs.org/aproba/-/aproba-1.2.0.tgz",
|
||||
"integrity": "sha512-Y9J6ZjXtoYh8RnXVCMOU/ttDmk1aBjunq9vO0ta5x85WDQiQfUF9sIPBITdbiiIVcBo03Hi3jMxigBtsddlXRw=="
|
||||
},
|
||||
"are-we-there-yet": {
|
||||
"version": "1.1.7",
|
||||
"resolved": "https://registry.npmjs.org/are-we-there-yet/-/are-we-there-yet-1.1.7.tgz",
|
||||
"integrity": "sha512-nxwy40TuMiUGqMyRHgCSWZ9FM4VAoRP4xUYSTv5ImRog+h9yISPbVH7H8fASCIzYn9wlEv4zvFL7uKDMCFQm3g==",
|
||||
"requires": {
|
||||
"delegates": "^1.0.0",
|
||||
"readable-stream": "^2.0.6"
|
||||
}
|
||||
},
|
||||
"asn1": {
|
||||
"version": "0.2.4",
|
||||
"resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.4.tgz",
|
||||
"integrity": "sha512-jxwzQpLQjSmWXgwaCZE9Nz+glAG01yF1QnWgbhGwHI5A6FRIEY6IVqtHhIepHqI7/kyEyQEagBC5mBEFlIYvdg==",
|
||||
"optional": true,
|
||||
"requires": {
|
||||
"safer-buffer": "~2.1.0"
|
||||
}
|
||||
},
|
||||
"assert-plus": {
|
||||
"version": "1.0.0",
|
||||
"resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz",
|
||||
"integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU=",
|
||||
"optional": true
|
||||
},
|
||||
"asynckit": {
|
||||
"version": "0.4.0",
|
||||
"resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz",
|
||||
"integrity": "sha1-x57Zf380y48robyXkLzDZkdLS3k=",
|
||||
"optional": true
|
||||
},
|
||||
"aws-sign2": {
|
||||
"version": "0.7.0",
|
||||
"resolved": "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.7.0.tgz",
|
||||
"integrity": "sha1-tG6JCTSpWR8tL2+G1+ap8bP+dqg=",
|
||||
"optional": true
|
||||
},
|
||||
"aws4": {
|
||||
"version": "1.11.0",
|
||||
"resolved": "https://registry.npmjs.org/aws4/-/aws4-1.11.0.tgz",
|
||||
"integrity": "sha512-xh1Rl34h6Fi1DC2WWKfxUTVqRsNnr6LsKz2+hfwDxQJWmrx8+c7ylaqBMcHfl1U1r2dsifOvKX3LQuLNZ+XSvA==",
|
||||
"optional": true
|
||||
},
|
||||
"balanced-match": {
|
||||
"version": "1.0.2",
|
||||
"resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz",
|
||||
"integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw=="
|
||||
},
|
||||
"bcrypt-pbkdf": {
|
||||
"version": "1.0.2",
|
||||
"resolved": "https://registry.npmjs.org/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz",
|
||||
"integrity": "sha1-pDAdOJtqQ/m2f/PKEaP2Y342Dp4=",
|
||||
"optional": true,
|
||||
"requires": {
|
||||
"tweetnacl": "^0.14.3"
|
||||
}
|
||||
},
|
||||
"block-stream": {
|
||||
"version": "0.0.9",
|
||||
"resolved": "https://registry.npmjs.org/block-stream/-/block-stream-0.0.9.tgz",
|
||||
"integrity": "sha1-E+v+d4oDIFz+A3UUgeu0szAMEmo=",
|
||||
"optional": true,
|
||||
"requires": {
|
||||
"inherits": "~2.0.0"
|
||||
}
|
||||
},
|
||||
"brace-expansion": {
|
||||
"version": "1.1.11",
|
||||
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz",
|
||||
"integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==",
|
||||
"requires": {
|
||||
"balanced-match": "^1.0.0",
|
||||
"concat-map": "0.0.1"
|
||||
}
|
||||
},
|
||||
"caseless": {
|
||||
"version": "0.12.0",
|
||||
"resolved": "https://registry.npmjs.org/caseless/-/caseless-0.12.0.tgz",
|
||||
"integrity": "sha1-G2gcIf+EAzyCZUMJBolCDRhxUdw=",
|
||||
"optional": true
|
||||
},
|
||||
"chownr": {
|
||||
"version": "1.1.4",
|
||||
"resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz",
|
||||
"integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg=="
|
||||
},
|
||||
"code-point-at": {
|
||||
"version": "1.1.0",
|
||||
"resolved": "https://registry.npmjs.org/code-point-at/-/code-point-at-1.1.0.tgz",
|
||||
"integrity": "sha1-DQcLTQQ6W+ozovGkDi7bPZpMz3c="
|
||||
},
|
||||
"combined-stream": {
|
||||
"version": "1.0.8",
|
||||
"resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz",
|
||||
"integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==",
|
||||
"optional": true,
|
||||
"requires": {
|
||||
"delayed-stream": "~1.0.0"
|
||||
}
|
||||
},
|
||||
"concat-map": {
|
||||
"version": "0.0.1",
|
||||
"resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz",
|
||||
"integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s="
|
||||
},
|
||||
"console-control-strings": {
|
||||
"version": "1.1.0",
|
||||
"resolved": "https://registry.npmjs.org/console-control-strings/-/console-control-strings-1.1.0.tgz",
|
||||
"integrity": "sha1-PXz0Rk22RG6mRL9LOVB/mFEAjo4="
|
||||
},
|
||||
"core-util-is": {
|
||||
"version": "1.0.3",
|
||||
"resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.3.tgz",
|
||||
"integrity": "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ=="
|
||||
},
|
||||
"dashdash": {
|
||||
"version": "1.14.1",
|
||||
"resolved": "https://registry.npmjs.org/dashdash/-/dashdash-1.14.1.tgz",
|
||||
"integrity": "sha1-hTz6D3y+L+1d4gMmuN1YEDX24vA=",
|
||||
"optional": true,
|
||||
"requires": {
|
||||
"assert-plus": "^1.0.0"
|
||||
}
|
||||
},
|
||||
"debug": {
|
||||
"version": "3.2.7",
|
||||
"resolved": "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz",
|
||||
"integrity": "sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==",
|
||||
"requires": {
|
||||
"ms": "^2.1.1"
|
||||
}
|
||||
},
|
||||
"deep-extend": {
|
||||
"version": "0.6.0",
|
||||
"resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz",
|
||||
"integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA=="
|
||||
},
|
||||
"delayed-stream": {
|
||||
"version": "1.0.0",
|
||||
"resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz",
|
||||
"integrity": "sha1-3zrhmayt+31ECqrgsp4icrJOxhk=",
|
||||
"optional": true
|
||||
},
|
||||
"delegates": {
|
||||
"version": "1.0.0",
|
||||
"resolved": "https://registry.npmjs.org/delegates/-/delegates-1.0.0.tgz",
|
||||
"integrity": "sha1-hMbhWbgZBP3KWaDvRM2HDTElD5o="
|
||||
},
|
||||
"detect-libc": {
|
||||
"version": "1.0.3",
|
||||
"resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-1.0.3.tgz",
|
||||
"integrity": "sha1-+hN8S9aY7fVc1c0CrFWfkaTEups="
|
||||
},
|
||||
"ecc-jsbn": {
|
||||
"version": "0.1.2",
|
||||
"resolved": "https://registry.npmjs.org/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz",
|
||||
"integrity": "sha1-OoOpBOVDUyh4dMVkt1SThoSamMk=",
|
||||
"optional": true,
|
||||
"requires": {
|
||||
"jsbn": "~0.1.0",
|
||||
"safer-buffer": "^2.1.0"
|
||||
}
|
||||
},
|
||||
"extend": {
|
||||
"version": "3.0.2",
|
||||
"resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz",
|
||||
"integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==",
|
||||
"optional": true
|
||||
},
|
||||
"extsprintf": {
|
||||
"version": "1.3.0",
|
||||
"resolved": "https://registry.npmjs.org/extsprintf/-/extsprintf-1.3.0.tgz",
|
||||
"integrity": "sha1-lpGEQOMEGnpBT4xS48V06zw+HgU=",
|
||||
"optional": true
|
||||
},
|
||||
"fast-deep-equal": {
|
||||
"version": "3.1.3",
|
||||
"resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz",
|
||||
"integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==",
|
||||
"optional": true
|
||||
},
|
||||
"fast-json-stable-stringify": {
|
||||
"version": "2.1.0",
|
||||
"resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz",
|
||||
"integrity": "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==",
|
||||
"optional": true
|
||||
},
|
||||
"forever-agent": {
|
||||
"version": "0.6.1",
|
||||
"resolved": "https://registry.npmjs.org/forever-agent/-/forever-agent-0.6.1.tgz",
|
||||
"integrity": "sha1-+8cfDEGt6zf5bFd60e1C2P2sypE=",
|
||||
"optional": true
|
||||
},
|
||||
"form-data": {
|
||||
"version": "2.3.3",
|
||||
"resolved": "https://registry.npmjs.org/form-data/-/form-data-2.3.3.tgz",
|
||||
"integrity": "sha512-1lLKB2Mu3aGP1Q/2eCOx0fNbRMe7XdwktwOruhfqqd0rIJWwN4Dh+E3hrPSlDCXnSR7UtZ1N38rVXm+6+MEhJQ==",
|
||||
"optional": true,
|
||||
"requires": {
|
||||
"asynckit": "^0.4.0",
|
||||
"combined-stream": "^1.0.6",
|
||||
"mime-types": "^2.1.12"
|
||||
}
|
||||
},
|
||||
"fs-extra": {
|
||||
"version": "10.0.0",
|
||||
"resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-10.0.0.tgz",
|
||||
"integrity": "sha512-C5owb14u9eJwizKGdchcDUQeFtlSHHthBk8pbX9Vc1PFZrLombudjDnNns88aYslCyF6IY5SUw3Roz6xShcEIQ==",
|
||||
"requires": {
|
||||
"graceful-fs": "^4.2.0",
|
||||
"jsonfile": "^6.0.1",
|
||||
"universalify": "^2.0.0"
|
||||
}
|
||||
},
|
||||
"fs-minipass": {
|
||||
"version": "1.2.7",
|
||||
"resolved": "https://registry.npmjs.org/fs-minipass/-/fs-minipass-1.2.7.tgz",
|
||||
"integrity": "sha512-GWSSJGFy4e9GUeCcbIkED+bgAoFyj7XF1mV8rma3QW4NIqX9Kyx79N/PF61H5udOV3aY1IaMLs6pGbH71nlCTA==",
|
||||
"requires": {
|
||||
"minipass": "^2.6.0"
|
||||
}
|
||||
},
|
||||
"fs.realpath": {
|
||||
"version": "1.0.0",
|
||||
"resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz",
|
||||
"integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8="
|
||||
},
|
||||
"fstream": {
|
||||
"version": "1.0.12",
|
||||
"resolved": "https://registry.npmjs.org/fstream/-/fstream-1.0.12.tgz",
|
||||
"integrity": "sha512-WvJ193OHa0GHPEL+AycEJgxvBEwyfRkN1vhjca23OaPVMCaLCXTd5qAu82AjTcgP1UJmytkOKb63Ypde7raDIg==",
|
||||
"optional": true,
|
||||
"requires": {
|
||||
"graceful-fs": "^4.1.2",
|
||||
"inherits": "~2.0.0",
|
||||
"mkdirp": ">=0.5 0",
|
||||
"rimraf": "2"
|
||||
}
|
||||
},
|
||||
"gauge": {
|
||||
"version": "2.7.4",
|
||||
"resolved": "https://registry.npmjs.org/gauge/-/gauge-2.7.4.tgz",
|
||||
"integrity": "sha1-LANAXHU4w51+s3sxcCLjJfsBi/c=",
|
||||
"requires": {
|
||||
"aproba": "^1.0.3",
|
||||
"console-control-strings": "^1.0.0",
|
||||
"has-unicode": "^2.0.0",
|
||||
"object-assign": "^4.1.0",
|
||||
"signal-exit": "^3.0.0",
|
||||
"string-width": "^1.0.1",
|
||||
"strip-ansi": "^3.0.1",
|
||||
"wide-align": "^1.1.0"
|
||||
}
|
||||
},
|
||||
"getpass": {
|
||||
"version": "0.1.7",
|
||||
"resolved": "https://registry.npmjs.org/getpass/-/getpass-0.1.7.tgz",
|
||||
"integrity": "sha1-Xv+OPmhNVprkyysSgmBOi6YhSfo=",
|
||||
"optional": true,
|
||||
"requires": {
|
||||
"assert-plus": "^1.0.0"
|
||||
}
|
||||
},
|
||||
"glob": {
|
||||
"version": "7.2.0",
|
||||
"resolved": "https://registry.npmjs.org/glob/-/glob-7.2.0.tgz",
|
||||
"integrity": "sha512-lmLf6gtyrPq8tTjSmrO94wBeQbFR3HbLHbuyD69wuyQkImp2hWqMGB47OX65FBkPffO641IP9jWa1z4ivqG26Q==",
|
||||
"requires": {
|
||||
"fs.realpath": "^1.0.0",
|
||||
"inflight": "^1.0.4",
|
||||
"inherits": "2",
|
||||
"minimatch": "^3.0.4",
|
||||
"once": "^1.3.0",
|
||||
"path-is-absolute": "^1.0.0"
|
||||
}
|
||||
},
|
||||
"graceful-fs": {
|
||||
"version": "4.2.8",
|
||||
"resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.8.tgz",
|
||||
"integrity": "sha512-qkIilPUYcNhJpd33n0GBXTB1MMPp14TxEsEs0pTrsSVucApsYzW5V+Q8Qxhik6KU3evy+qkAAowTByymK0avdg=="
|
||||
},
|
||||
"har-schema": {
|
||||
"version": "2.0.0",
|
||||
"resolved": "https://registry.npmjs.org/har-schema/-/har-schema-2.0.0.tgz",
|
||||
"integrity": "sha1-qUwiJOvKwEeCoNkDVSHyRzW37JI=",
|
||||
"optional": true
|
||||
},
|
||||
"har-validator": {
|
||||
"version": "5.1.5",
|
||||
"resolved": "https://registry.npmjs.org/har-validator/-/har-validator-5.1.5.tgz",
|
||||
"integrity": "sha512-nmT2T0lljbxdQZfspsno9hgrG3Uir6Ks5afism62poxqBM6sDnMEuPmzTq8XN0OEwqKLLdh1jQI3qyE66Nzb3w==",
|
||||
"optional": true,
|
||||
"requires": {
|
||||
"ajv": "^6.12.3",
|
||||
"har-schema": "^2.0.0"
|
||||
}
|
||||
},
|
||||
"has-unicode": {
|
||||
"version": "2.0.1",
|
||||
"resolved": "https://registry.npmjs.org/has-unicode/-/has-unicode-2.0.1.tgz",
|
||||
"integrity": "sha1-4Ob+aijPUROIVeCG0Wkedx3iqLk="
|
||||
},
|
||||
"http-signature": {
|
||||
"version": "1.2.0",
|
||||
"resolved": "https://registry.npmjs.org/http-signature/-/http-signature-1.2.0.tgz",
|
||||
"integrity": "sha1-muzZJRFHcvPZW2WmCruPfBj7rOE=",
|
||||
"optional": true,
|
||||
"requires": {
|
||||
"assert-plus": "^1.0.0",
|
||||
"jsprim": "^1.2.2",
|
||||
"sshpk": "^1.7.0"
|
||||
}
|
||||
},
|
||||
"iconv-lite": {
|
||||
"version": "0.4.24",
|
||||
"resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz",
|
||||
"integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==",
|
||||
"requires": {
|
||||
"safer-buffer": ">= 2.1.2 < 3"
|
||||
}
|
||||
},
|
||||
"ignore-walk": {
|
||||
"version": "3.0.4",
|
||||
"resolved": "https://registry.npmjs.org/ignore-walk/-/ignore-walk-3.0.4.tgz",
|
||||
"integrity": "sha512-PY6Ii8o1jMRA1z4F2hRkH/xN59ox43DavKvD3oDpfurRlOJyAHpifIwpbdv1n4jt4ov0jSpw3kQ4GhJnpBL6WQ==",
|
||||
"requires": {
|
||||
"minimatch": "^3.0.4"
|
||||
}
|
||||
},
|
||||
"inflight": {
|
||||
"version": "1.0.6",
|
||||
"resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz",
|
||||
"integrity": "sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=",
|
||||
"requires": {
|
||||
"once": "^1.3.0",
|
||||
"wrappy": "1"
|
||||
}
|
||||
},
|
||||
"inherits": {
|
||||
"version": "2.0.4",
|
||||
"resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz",
|
||||
"integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ=="
|
||||
},
|
||||
"ini": {
|
||||
"version": "1.3.8",
|
||||
"resolved": "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz",
|
||||
"integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew=="
|
||||
},
|
||||
"is-fullwidth-code-point": {
|
||||
"version": "1.0.0",
|
||||
"resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-1.0.0.tgz",
|
||||
"integrity": "sha1-754xOG8DGn8NZDr4L95QxFfvAMs=",
|
||||
"requires": {
|
||||
"number-is-nan": "^1.0.0"
|
||||
}
|
||||
},
|
||||
"is-typedarray": {
|
||||
"version": "1.0.0",
|
||||
"resolved": "https://registry.npmjs.org/is-typedarray/-/is-typedarray-1.0.0.tgz",
|
||||
"integrity": "sha1-5HnICFjfDBsR3dppQPlgEfzaSpo=",
|
||||
"optional": true
|
||||
},
|
||||
"isarray": {
|
||||
"version": "1.0.0",
|
||||
"resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz",
|
||||
"integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE="
|
||||
},
|
||||
"isexe": {
|
||||
"version": "2.0.0",
|
||||
"resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz",
|
||||
"integrity": "sha1-6PvzdNxVb/iUehDcsFctYz8s+hA=",
|
||||
"optional": true
|
||||
},
|
||||
"isstream": {
|
||||
"version": "0.1.2",
|
||||
"resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz",
|
||||
"integrity": "sha1-R+Y/evVa+m+S4VAOaQ64uFKcCZo=",
|
||||
"optional": true
|
||||
},
|
||||
"jsbn": {
|
||||
"version": "0.1.1",
|
||||
"resolved": "https://registry.npmjs.org/jsbn/-/jsbn-0.1.1.tgz",
|
||||
"integrity": "sha1-peZUwuWi3rXyAdls77yoDA7y9RM=",
|
||||
"optional": true
|
||||
},
|
||||
"json-schema": {
|
||||
"version": "0.2.3",
|
||||
"resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.2.3.tgz",
|
||||
"integrity": "sha1-tIDIkuWaLwWVTOcnvT8qTogvnhM=",
|
||||
"optional": true
|
||||
},
|
||||
"json-schema-traverse": {
|
||||
"version": "0.4.1",
|
||||
"resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz",
|
||||
"integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==",
|
||||
"optional": true
|
||||
},
|
||||
"json-stringify-safe": {
|
||||
"version": "5.0.1",
|
||||
"resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz",
|
||||
"integrity": "sha1-Epai1Y/UXxmg9s4B1lcB4sc1tus=",
|
||||
"optional": true
|
||||
},
|
||||
"jsonfile": {
|
||||
"version": "6.1.0",
|
||||
"resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-6.1.0.tgz",
|
||||
"integrity": "sha512-5dgndWOriYSm5cnYaJNhalLNDKOqFwyDB/rr1E9ZsGciGvKPs8R2xYGCacuf3z6K1YKDz182fd+fY3cn3pMqXQ==",
|
||||
"requires": {
|
||||
"graceful-fs": "^4.1.6",
|
||||
"universalify": "^2.0.0"
|
||||
}
|
||||
},
|
||||
"jsprim": {
|
||||
"version": "1.4.1",
|
||||
"resolved": "https://registry.npmjs.org/jsprim/-/jsprim-1.4.1.tgz",
|
||||
"integrity": "sha1-MT5mvB5cwG5Di8G3SZwuXFastqI=",
|
||||
"optional": true,
|
||||
"requires": {
|
||||
"assert-plus": "1.0.0",
|
||||
"extsprintf": "1.3.0",
|
||||
"json-schema": "0.2.3",
|
||||
"verror": "1.10.0"
|
||||
}
|
||||
},
|
||||
"mime-db": {
|
||||
"version": "1.49.0",
|
||||
"resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.49.0.tgz",
|
||||
"integrity": "sha512-CIc8j9URtOVApSFCQIF+VBkX1RwXp/oMMOrqdyXSBXq5RWNEsRfyj1kiRnQgmNXmHxPoFIxOroKA3zcU9P+nAA==",
|
||||
"optional": true
|
||||
},
|
||||
"mime-types": {
|
||||
"version": "2.1.32",
|
||||
"resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.32.tgz",
|
||||
"integrity": "sha512-hJGaVS4G4c9TSMYh2n6SQAGrC4RnfU+daP8G7cSCmaqNjiOoUY0VHCMS42pxnQmVF1GWwFhbHWn3RIxCqTmZ9A==",
|
||||
"optional": true,
|
||||
"requires": {
|
||||
"mime-db": "1.49.0"
|
||||
}
|
||||
},
|
||||
"minimatch": {
|
||||
"version": "3.0.4",
|
||||
"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz",
|
||||
"integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==",
|
||||
"requires": {
|
||||
"brace-expansion": "^1.1.7"
|
||||
}
|
||||
},
|
||||
"minimist": {
|
||||
"version": "1.2.5",
|
||||
"resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.5.tgz",
|
||||
"integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw=="
|
||||
},
|
||||
"minipass": {
|
||||
"version": "2.9.0",
|
||||
"resolved": "https://registry.npmjs.org/minipass/-/minipass-2.9.0.tgz",
|
||||
"integrity": "sha512-wxfUjg9WebH+CUDX/CdbRlh5SmfZiy/hpkxaRI16Y9W56Pa75sWgd/rvFilSgrauD9NyFymP/+JFV3KwzIsJeg==",
|
||||
"requires": {
|
||||
"safe-buffer": "^5.1.2",
|
||||
"yallist": "^3.0.0"
|
||||
}
|
||||
},
|
||||
"minizlib": {
|
||||
"version": "1.3.3",
|
||||
"resolved": "https://registry.npmjs.org/minizlib/-/minizlib-1.3.3.tgz",
|
||||
"integrity": "sha512-6ZYMOEnmVsdCeTJVE0W9ZD+pVnE8h9Hma/iOwwRDsdQoePpoX56/8B6z3P9VNwppJuBKNRuFDRNRqRWexT9G9Q==",
|
||||
"requires": {
|
||||
"minipass": "^2.9.0"
|
||||
}
|
||||
},
|
||||
"mkdirp": {
|
||||
"version": "0.5.5",
|
||||
"resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.5.tgz",
|
||||
"integrity": "sha512-NKmAlESf6jMGym1++R0Ra7wvhV+wFW63FaSOFPwRahvea0gMUcGUhVeAg/0BC0wiv9ih5NYPB1Wn1UEI1/L+xQ==",
|
||||
"requires": {
|
||||
"minimist": "^1.2.5"
|
||||
}
|
||||
},
|
||||
"ms": {
|
||||
"version": "2.1.3",
|
||||
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",
|
||||
"integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="
|
||||
},
|
||||
"needle": {
|
||||
"version": "2.9.1",
|
||||
"resolved": "https://registry.npmjs.org/needle/-/needle-2.9.1.tgz",
|
||||
"integrity": "sha512-6R9fqJ5Zcmf+uYaFgdIHmLwNldn5HbK8L5ybn7Uz+ylX/rnOsSp1AHcvQSrCaFN+qNM1wpymHqD7mVasEOlHGQ==",
|
||||
"requires": {
|
||||
"debug": "^3.2.6",
|
||||
"iconv-lite": "^0.4.4",
|
||||
"sax": "^1.2.4"
|
||||
}
|
||||
},
|
||||
"node-addon-api": {
|
||||
"version": "3.2.1",
|
||||
"resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-3.2.1.tgz",
|
||||
"integrity": "sha512-mmcei9JghVNDYydghQmeDX8KoAm0FAiYyIcUt/N4nhyAipB17pllZQDOJD2fotxABnt4Mdz+dKTO7eftLg4d0A=="
|
||||
},
|
||||
"node-gyp": {
|
||||
"version": "3.8.0",
|
||||
"resolved": "https://registry.npmjs.org/node-gyp/-/node-gyp-3.8.0.tgz",
|
||||
"integrity": "sha512-3g8lYefrRRzvGeSowdJKAKyks8oUpLEd/DyPV4eMhVlhJ0aNaZqIrNUIPuEWWTAoPqyFkfGrM67MC69baqn6vA==",
|
||||
"optional": true,
|
||||
"requires": {
|
||||
"fstream": "^1.0.0",
|
||||
"glob": "^7.0.3",
|
||||
"graceful-fs": "^4.1.2",
|
||||
"mkdirp": "^0.5.0",
|
||||
"nopt": "2 || 3",
|
||||
"npmlog": "0 || 1 || 2 || 3 || 4",
|
||||
"osenv": "0",
|
||||
"request": "^2.87.0",
|
||||
"rimraf": "2",
|
||||
"semver": "~5.3.0",
|
||||
"tar": "^2.0.0",
|
||||
"which": "1"
|
||||
}
|
||||
},
|
||||
"node-pre-gyp": {
|
||||
"version": "0.11.0",
|
||||
"resolved": "https://registry.npmjs.org/node-pre-gyp/-/node-pre-gyp-0.11.0.tgz",
|
||||
"integrity": "sha512-TwWAOZb0j7e9eGaf9esRx3ZcLaE5tQ2lvYy1pb5IAaG1a2e2Kv5Lms1Y4hpj+ciXJRofIxxlt5haeQ/2ANeE0Q==",
|
||||
"requires": {
|
||||
"detect-libc": "^1.0.2",
|
||||
"mkdirp": "^0.5.1",
|
||||
"needle": "^2.2.1",
|
||||
"nopt": "^4.0.1",
|
||||
"npm-packlist": "^1.1.6",
|
||||
"npmlog": "^4.0.2",
|
||||
"rc": "^1.2.7",
|
||||
"rimraf": "^2.6.1",
|
||||
"semver": "^5.3.0",
|
||||
"tar": "^4"
|
||||
},
|
||||
"dependencies": {
|
||||
"nopt": {
|
||||
"version": "4.0.3",
|
||||
"resolved": "https://registry.npmjs.org/nopt/-/nopt-4.0.3.tgz",
|
||||
"integrity": "sha512-CvaGwVMztSMJLOeXPrez7fyfObdZqNUK1cPAEzLHrTybIua9pMdmmPR5YwtfNftIOMv3DPUhFaxsZMNTQO20Kg==",
|
||||
"requires": {
|
||||
"abbrev": "1",
|
||||
"osenv": "^0.1.4"
|
||||
}
|
||||
},
|
||||
"safe-buffer": {
|
||||
"version": "5.2.1",
|
||||
"resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz",
|
||||
"integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ=="
|
||||
},
|
||||
"tar": {
|
||||
"version": "4.4.19",
|
||||
"resolved": "https://registry.npmjs.org/tar/-/tar-4.4.19.tgz",
|
||||
"integrity": "sha512-a20gEsvHnWe0ygBY8JbxoM4w3SJdhc7ZAuxkLqh+nvNQN2IOt0B5lLgM490X5Hl8FF0dl0tOf2ewFYAlIFgzVA==",
|
||||
"requires": {
|
||||
"chownr": "^1.1.4",
|
||||
"fs-minipass": "^1.2.7",
|
||||
"minipass": "^2.9.0",
|
||||
"minizlib": "^1.3.3",
|
||||
"mkdirp": "^0.5.5",
|
||||
"safe-buffer": "^5.2.1",
|
||||
"yallist": "^3.1.1"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"nopt": {
|
||||
"version": "3.0.6",
|
||||
"resolved": "https://registry.npmjs.org/nopt/-/nopt-3.0.6.tgz",
|
||||
"integrity": "sha1-xkZdvwirzU2zWTF/eaxopkayj/k=",
|
||||
"optional": true,
|
||||
"requires": {
|
||||
"abbrev": "1"
|
||||
}
|
||||
},
|
||||
"npm-bundled": {
|
||||
"version": "1.1.2",
|
||||
"resolved": "https://registry.npmjs.org/npm-bundled/-/npm-bundled-1.1.2.tgz",
|
||||
"integrity": "sha512-x5DHup0SuyQcmL3s7Rx/YQ8sbw/Hzg0rj48eN0dV7hf5cmQq5PXIeioroH3raV1QC1yh3uTYuMThvEQF3iKgGQ==",
|
||||
"requires": {
|
||||
"npm-normalize-package-bin": "^1.0.1"
|
||||
}
|
||||
},
|
||||
"npm-normalize-package-bin": {
|
||||
"version": "1.0.1",
|
||||
"resolved": "https://registry.npmjs.org/npm-normalize-package-bin/-/npm-normalize-package-bin-1.0.1.tgz",
|
||||
"integrity": "sha512-EPfafl6JL5/rU+ot6P3gRSCpPDW5VmIzX959Ob1+ySFUuuYHWHekXpwdUZcKP5C+DS4GEtdJluwBjnsNDl+fSA=="
|
||||
},
|
||||
"npm-packlist": {
|
||||
"version": "1.4.8",
|
||||
"resolved": "https://registry.npmjs.org/npm-packlist/-/npm-packlist-1.4.8.tgz",
|
||||
"integrity": "sha512-5+AZgwru5IevF5ZdnFglB5wNlHG1AOOuw28WhUq8/8emhBmLv6jX5by4WJCh7lW0uSYZYS6DXqIsyZVIXRZU9A==",
|
||||
"requires": {
|
||||
"ignore-walk": "^3.0.1",
|
||||
"npm-bundled": "^1.0.1",
|
||||
"npm-normalize-package-bin": "^1.0.1"
|
||||
}
|
||||
},
|
||||
"npmlog": {
|
||||
"version": "4.1.2",
|
||||
"resolved": "https://registry.npmjs.org/npmlog/-/npmlog-4.1.2.tgz",
|
||||
"integrity": "sha512-2uUqazuKlTaSI/dC8AzicUck7+IrEaOnN/e0jd3Xtt1KcGpwx30v50mL7oPyr/h9bL3E4aZccVwpwP+5W9Vjkg==",
|
||||
"requires": {
|
||||
"are-we-there-yet": "~1.1.2",
|
||||
"console-control-strings": "~1.1.0",
|
||||
"gauge": "~2.7.3",
|
||||
"set-blocking": "~2.0.0"
|
||||
}
|
||||
},
|
||||
"number-is-nan": {
|
||||
"version": "1.0.1",
|
||||
"resolved": "https://registry.npmjs.org/number-is-nan/-/number-is-nan-1.0.1.tgz",
|
||||
"integrity": "sha1-CXtgK1NCKlIsGvuHkDGDNpQaAR0="
|
||||
},
|
||||
"oauth-sign": {
|
||||
"version": "0.9.0",
|
||||
"resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.9.0.tgz",
|
||||
"integrity": "sha512-fexhUFFPTGV8ybAtSIGbV6gOkSv8UtRbDBnAyLQw4QPKkgNlsH2ByPGtMUqdWkos6YCRmAqViwgZrJc/mRDzZQ==",
|
||||
"optional": true
|
||||
},
|
||||
"object-assign": {
|
||||
"version": "4.1.1",
|
||||
"resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz",
|
||||
"integrity": "sha1-IQmtx5ZYh8/AXLvUQsrIv7s2CGM="
|
||||
},
|
||||
"once": {
|
||||
"version": "1.4.0",
|
||||
"resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz",
|
||||
"integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=",
|
||||
"requires": {
|
||||
"wrappy": "1"
|
||||
}
|
||||
},
|
||||
"os-homedir": {
|
||||
"version": "1.0.2",
|
||||
"resolved": "https://registry.npmjs.org/os-homedir/-/os-homedir-1.0.2.tgz",
|
||||
"integrity": "sha1-/7xJiDNuDoM94MFox+8VISGqf7M="
|
||||
},
|
||||
"os-tmpdir": {
|
||||
"version": "1.0.2",
|
||||
"resolved": "https://registry.npmjs.org/os-tmpdir/-/os-tmpdir-1.0.2.tgz",
|
||||
"integrity": "sha1-u+Z0BseaqFxc/sdm/lc0VV36EnQ="
|
||||
},
|
||||
"osenv": {
|
||||
"version": "0.1.5",
|
||||
"resolved": "https://registry.npmjs.org/osenv/-/osenv-0.1.5.tgz",
|
||||
"integrity": "sha512-0CWcCECdMVc2Rw3U5w9ZjqX6ga6ubk1xDVKxtBQPK7wis/0F2r9T6k4ydGYhecl7YUBxBVxhL5oisPsNxAPe2g==",
|
||||
"requires": {
|
||||
"os-homedir": "^1.0.0",
|
||||
"os-tmpdir": "^1.0.0"
|
||||
}
|
||||
},
|
||||
"path-is-absolute": {
|
||||
"version": "1.0.1",
|
||||
"resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz",
|
||||
"integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18="
|
||||
},
|
||||
"performance-now": {
|
||||
"version": "2.1.0",
|
||||
"resolved": "https://registry.npmjs.org/performance-now/-/performance-now-2.1.0.tgz",
|
||||
"integrity": "sha1-Ywn04OX6kT7BxpMHrjZLSzd8nns=",
|
||||
"optional": true
|
||||
},
|
||||
"process-nextick-args": {
|
||||
"version": "2.0.1",
|
||||
"resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz",
|
||||
"integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag=="
|
||||
},
|
||||
"psl": {
|
||||
"version": "1.8.0",
|
||||
"resolved": "https://registry.npmjs.org/psl/-/psl-1.8.0.tgz",
|
||||
"integrity": "sha512-RIdOzyoavK+hA18OGGWDqUTsCLhtA7IcZ/6NCs4fFJaHBDab+pDDmDIByWFRQJq2Cd7r1OoQxBGKOaztq+hjIQ==",
|
||||
"optional": true
|
||||
},
|
||||
"punycode": {
|
||||
"version": "2.1.1",
|
||||
"resolved": "https://registry.npmjs.org/punycode/-/punycode-2.1.1.tgz",
|
||||
"integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==",
|
||||
"optional": true
|
||||
},
|
||||
"qs": {
|
||||
"version": "6.5.2",
|
||||
"resolved": "https://registry.npmjs.org/qs/-/qs-6.5.2.tgz",
|
||||
"integrity": "sha512-N5ZAX4/LxJmF+7wN74pUD6qAh9/wnvdQcjq9TZjevvXzSUo7bfmw91saqMjzGS2xq91/odN2dW/WOl7qQHNDGA==",
|
||||
"optional": true
|
||||
},
|
||||
"rc": {
|
||||
"version": "1.2.8",
|
||||
"resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz",
|
||||
"integrity": "sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==",
|
||||
"requires": {
|
||||
"deep-extend": "^0.6.0",
|
||||
"ini": "~1.3.0",
|
||||
"minimist": "^1.2.0",
|
||||
"strip-json-comments": "~2.0.1"
|
||||
}
|
||||
},
|
||||
"readable-stream": {
|
||||
"version": "2.3.7",
|
||||
"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz",
|
||||
"integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==",
|
||||
"requires": {
|
||||
"core-util-is": "~1.0.0",
|
||||
"inherits": "~2.0.3",
|
||||
"isarray": "~1.0.0",
|
||||
"process-nextick-args": "~2.0.0",
|
||||
"safe-buffer": "~5.1.1",
|
||||
"string_decoder": "~1.1.1",
|
||||
"util-deprecate": "~1.0.1"
|
||||
}
|
||||
},
|
||||
"request": {
|
||||
"version": "2.88.2",
|
||||
"resolved": "https://registry.npmjs.org/request/-/request-2.88.2.tgz",
|
||||
"integrity": "sha512-MsvtOrfG9ZcrOwAW+Qi+F6HbD0CWXEh9ou77uOb7FM2WPhwT7smM833PzanhJLsgXjN89Ir6V2PczXNnMpwKhw==",
|
||||
"optional": true,
|
||||
"requires": {
|
||||
"aws-sign2": "~0.7.0",
|
||||
"aws4": "^1.8.0",
|
||||
"caseless": "~0.12.0",
|
||||
"combined-stream": "~1.0.6",
|
||||
"extend": "~3.0.2",
|
||||
"forever-agent": "~0.6.1",
|
||||
"form-data": "~2.3.2",
|
||||
"har-validator": "~5.1.3",
|
||||
"http-signature": "~1.2.0",
|
||||
"is-typedarray": "~1.0.0",
|
||||
"isstream": "~0.1.2",
|
||||
"json-stringify-safe": "~5.0.1",
|
||||
"mime-types": "~2.1.19",
|
||||
"oauth-sign": "~0.9.0",
|
||||
"performance-now": "^2.1.0",
|
||||
"qs": "~6.5.2",
|
||||
"safe-buffer": "^5.1.2",
|
||||
"tough-cookie": "~2.5.0",
|
||||
"tunnel-agent": "^0.6.0",
|
||||
"uuid": "^3.3.2"
|
||||
}
|
||||
},
|
||||
"rimraf": {
|
||||
"version": "2.7.1",
|
||||
"resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.7.1.tgz",
|
||||
"integrity": "sha512-uWjbaKIK3T1OSVptzX7Nl6PvQ3qAGtKEtVRjRuazjfL3Bx5eI409VZSqgND+4UNnmzLVdPj9FqFJNPqBZFve4w==",
|
||||
"requires": {
|
||||
"glob": "^7.1.3"
|
||||
}
|
||||
},
|
||||
"safe-buffer": {
|
||||
"version": "5.1.2",
|
||||
"resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz",
|
||||
"integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g=="
|
||||
},
|
||||
"safer-buffer": {
|
||||
"version": "2.1.2",
|
||||
"resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz",
|
||||
"integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg=="
|
||||
},
|
||||
"sax": {
|
||||
"version": "1.2.4",
|
||||
"resolved": "https://registry.npmjs.org/sax/-/sax-1.2.4.tgz",
|
||||
"integrity": "sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw=="
|
||||
},
|
||||
"semver": {
|
||||
"version": "5.3.0",
|
||||
"resolved": "https://registry.npmjs.org/semver/-/semver-5.3.0.tgz",
|
||||
"integrity": "sha1-myzl094C0XxgEq0yaqa00M9U+U8="
|
||||
},
|
||||
"set-blocking": {
|
||||
"version": "2.0.0",
|
||||
"resolved": "https://registry.npmjs.org/set-blocking/-/set-blocking-2.0.0.tgz",
|
||||
"integrity": "sha1-BF+XgtARrppoA93TgrJDkrPYkPc="
|
||||
},
|
||||
"signal-exit": {
|
||||
"version": "3.0.4",
|
||||
"resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.4.tgz",
|
||||
"integrity": "sha512-rqYhcAnZ6d/vTPGghdrw7iumdcbXpsk1b8IG/rz+VWV51DM0p7XCtMoJ3qhPLIbp3tvyt3pKRbaaEMZYpHto8Q=="
|
||||
},
|
||||
"sqlite": {
|
||||
"version": "4.0.23",
|
||||
"resolved": "https://registry.npmjs.org/sqlite/-/sqlite-4.0.23.tgz",
|
||||
"integrity": "sha512-dSdmSkrdIhUL7xP/fiEMfFuAo4dxb0afag3rK8T4Y9lYxE3g3fXT0J8H9qSFvmcKxnM0zEA8yvLbpdWQ8mom3g=="
|
||||
},
|
||||
"sqlite3": {
|
||||
"version": "5.0.2",
|
||||
"resolved": "https://registry.npmjs.org/sqlite3/-/sqlite3-5.0.2.tgz",
|
||||
"integrity": "sha512-1SdTNo+BVU211Xj1csWa8lV6KM0CtucDwRyA0VHl91wEH1Mgh7RxUpI4rVvG7OhHrzCSGaVyW5g8vKvlrk9DJA==",
|
||||
"requires": {
|
||||
"node-addon-api": "^3.0.0",
|
||||
"node-gyp": "3.x",
|
||||
"node-pre-gyp": "^0.11.0"
|
||||
}
|
||||
},
|
||||
"sshpk": {
|
||||
"version": "1.16.1",
|
||||
"resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.16.1.tgz",
|
||||
"integrity": "sha512-HXXqVUq7+pcKeLqqZj6mHFUMvXtOJt1uoUx09pFW6011inTMxqI8BA8PM95myrIyyKwdnzjdFjLiE6KBPVtJIg==",
|
||||
"optional": true,
|
||||
"requires": {
|
||||
"asn1": "~0.2.3",
|
||||
"assert-plus": "^1.0.0",
|
||||
"bcrypt-pbkdf": "^1.0.0",
|
||||
"dashdash": "^1.12.0",
|
||||
"ecc-jsbn": "~0.1.1",
|
||||
"getpass": "^0.1.1",
|
||||
"jsbn": "~0.1.0",
|
||||
"safer-buffer": "^2.0.2",
|
||||
"tweetnacl": "~0.14.0"
|
||||
}
|
||||
},
|
||||
"string-width": {
|
||||
"version": "1.0.2",
|
||||
"resolved": "https://registry.npmjs.org/string-width/-/string-width-1.0.2.tgz",
|
||||
"integrity": "sha1-EYvfW4zcUaKn5w0hHgfisLmxB9M=",
|
||||
"requires": {
|
||||
"code-point-at": "^1.0.0",
|
||||
"is-fullwidth-code-point": "^1.0.0",
|
||||
"strip-ansi": "^3.0.0"
|
||||
}
|
||||
},
|
||||
"string_decoder": {
|
||||
"version": "1.1.1",
|
||||
"resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz",
|
||||
"integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==",
|
||||
"requires": {
|
||||
"safe-buffer": "~5.1.0"
|
||||
}
|
||||
},
|
||||
"strip-ansi": {
|
||||
"version": "3.0.1",
|
||||
"resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz",
|
||||
"integrity": "sha1-ajhfuIU9lS1f8F0Oiq+UJ43GPc8=",
|
||||
"requires": {
|
||||
"ansi-regex": "^2.0.0"
|
||||
}
|
||||
},
|
||||
"strip-json-comments": {
|
||||
"version": "2.0.1",
|
||||
"resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz",
|
||||
"integrity": "sha1-PFMZQukIwml8DsNEhYwobHygpgo="
|
||||
},
|
||||
"tar": {
|
||||
"version": "2.2.2",
|
||||
"resolved": "https://registry.npmjs.org/tar/-/tar-2.2.2.tgz",
|
||||
"integrity": "sha512-FCEhQ/4rE1zYv9rYXJw/msRqsnmlje5jHP6huWeBZ704jUTy02c5AZyWujpMR1ax6mVw9NyJMfuK2CMDWVIfgA==",
|
||||
"optional": true,
|
||||
"requires": {
|
||||
"block-stream": "*",
|
||||
"fstream": "^1.0.12",
|
||||
"inherits": "2"
|
||||
}
|
||||
},
|
||||
"tough-cookie": {
|
||||
"version": "2.5.0",
|
||||
"resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.5.0.tgz",
|
||||
"integrity": "sha512-nlLsUzgm1kfLXSXfRZMc1KLAugd4hqJHDTvc2hDIwS3mZAfMEuMbc03SujMF+GEcpaX/qboeycw6iO8JwVv2+g==",
|
||||
"optional": true,
|
||||
"requires": {
|
||||
"psl": "^1.1.28",
|
||||
"punycode": "^2.1.1"
|
||||
}
|
||||
},
|
||||
"tunnel-agent": {
|
||||
"version": "0.6.0",
|
||||
"resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz",
|
||||
"integrity": "sha1-J6XeoGs2sEoKmWZ3SykIaPD8QP0=",
|
||||
"optional": true,
|
||||
"requires": {
|
||||
"safe-buffer": "^5.0.1"
|
||||
}
|
||||
},
|
||||
"tweetnacl": {
|
||||
"version": "0.14.5",
|
||||
"resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-0.14.5.tgz",
|
||||
"integrity": "sha1-WuaBd/GS1EViadEIr6k/+HQ/T2Q=",
|
||||
"optional": true
|
||||
},
|
||||
"universalify": {
|
||||
"version": "2.0.0",
|
||||
"resolved": "https://registry.npmjs.org/universalify/-/universalify-2.0.0.tgz",
|
||||
"integrity": "sha512-hAZsKq7Yy11Zu1DE0OzWjw7nnLZmJZYTDZZyEFHZdUhV8FkH5MCfoU1XMaxXovpyW5nq5scPqq0ZDP9Zyl04oQ=="
|
||||
},
|
||||
"uri-js": {
|
||||
"version": "4.4.1",
|
||||
"resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.4.1.tgz",
|
||||
"integrity": "sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==",
|
||||
"optional": true,
|
||||
"requires": {
|
||||
"punycode": "^2.1.0"
|
||||
}
|
||||
},
|
||||
"util-deprecate": {
|
||||
"version": "1.0.2",
|
||||
"resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz",
|
||||
"integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8="
|
||||
},
|
||||
"uuid": {
|
||||
"version": "3.4.0",
|
||||
"resolved": "https://registry.npmjs.org/uuid/-/uuid-3.4.0.tgz",
|
||||
"integrity": "sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==",
|
||||
"optional": true
|
||||
},
|
||||
"verror": {
|
||||
"version": "1.10.0",
|
||||
"resolved": "https://registry.npmjs.org/verror/-/verror-1.10.0.tgz",
|
||||
"integrity": "sha1-OhBcoXBTr1XW4nDB+CiGguGNpAA=",
|
||||
"optional": true,
|
||||
"requires": {
|
||||
"assert-plus": "^1.0.0",
|
||||
"core-util-is": "1.0.2",
|
||||
"extsprintf": "^1.2.0"
|
||||
},
|
||||
"dependencies": {
|
||||
"core-util-is": {
|
||||
"version": "1.0.2",
|
||||
"resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz",
|
||||
"integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=",
|
||||
"optional": true
|
||||
}
|
||||
}
|
||||
},
|
||||
"which": {
|
||||
"version": "1.3.1",
|
||||
"resolved": "https://registry.npmjs.org/which/-/which-1.3.1.tgz",
|
||||
"integrity": "sha512-HxJdYWq1MTIQbJ3nw0cqssHoTNU267KlrDuGZ1WYlxDStUtKUhOaJmh112/TZmHxxUfuJqPXSOm7tDyas0OSIQ==",
|
||||
"optional": true,
|
||||
"requires": {
|
||||
"isexe": "^2.0.0"
|
||||
}
|
||||
},
|
||||
"wide-align": {
|
||||
"version": "1.1.3",
|
||||
"resolved": "https://registry.npmjs.org/wide-align/-/wide-align-1.1.3.tgz",
|
||||
"integrity": "sha512-QGkOQc8XL6Bt5PwnsExKBPuMKBxnGxWWW3fU55Xt4feHozMUhdUMaBCk290qpm/wG5u/RSKzwdAC4i51YigihA==",
|
||||
"requires": {
|
||||
"string-width": "^1.0.2 || 2"
|
||||
}
|
||||
},
|
||||
"wrappy": {
|
||||
"version": "1.0.2",
|
||||
"resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz",
|
||||
"integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8="
|
||||
},
|
||||
"yallist": {
|
||||
"version": "3.1.1",
|
||||
"resolved": "https://registry.npmjs.org/yallist/-/yallist-3.1.1.tgz",
|
||||
"integrity": "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g=="
|
||||
}
|
||||
}
|
||||
}
|
||||
17
06-output-generator/package.json
Normal file
17
06-output-generator/package.json
Normal file
@ -0,0 +1,17 @@
|
||||
{
|
||||
"name": "06-output-generator",
|
||||
"version": "1.0.0",
|
||||
"description": "",
|
||||
"main": "01-output-generator.js",
|
||||
"dependencies": {
|
||||
"fs-extra": "^10.0.0",
|
||||
"sqlite": "^4.0.23",
|
||||
"sqlite3": "^5.0.2"
|
||||
},
|
||||
"devDependencies": {},
|
||||
"scripts": {
|
||||
"test": "echo \"Error: no test specified\" && exit 1"
|
||||
},
|
||||
"author": "",
|
||||
"license": "ISC"
|
||||
}
|
||||
274
README.md
Normal file
274
README.md
Normal file
@ -0,0 +1,274 @@
|
||||
# ultimate-guitar.com Tab Scraper
|
||||
|
||||
This file set allows for a 6 step process to scrape the tabs off ultimate-guitar.com.
|
||||
|
||||
It takes advantage of a "feature" in the ultimate-guitar.com's rendering techniques that puts website data inside of a div with class '.js-store'.
|
||||
|
||||
Scraping all 1.1 million public tabs from the site is pretty easy and can be done in 6 steps.
|
||||
|
||||
You're going to want a VPN for this because you will get kicked off and IP blocked every 2-8 hours (depending on how agressive you are when scraping). Reconnect to another IP and you'll be good to continue scraping.
|
||||
|
||||
Download the sqlite3 command line client from https://sqlite.org/download.html
|
||||
Download Node.js from https://nodejs.org/
|
||||
|
||||
## 1. Scrape Tab URLs
|
||||
|
||||
This step maps out all pages on ultimate-guitar.com that can be scraped
|
||||
|
||||
Enter 01-scraper-urls
|
||||
|
||||
Run
|
||||
> npm install
|
||||
|
||||
Open up 01-scrape-bands.js and customize the band list links
|
||||
|
||||
Run
|
||||
> node 01-scrape-bands.js
|
||||
|
||||
This script will save the artist data to output/artists/*.json
|
||||
|
||||
Open up 02-scrape-artist-tab-urls.js and customize the artist file list
|
||||
|
||||
Run
|
||||
> node 02-scrape-artist-tab-urls.js
|
||||
|
||||
This script will add tab information to the artist data and save it to output/artists-with-tabs/*.json
|
||||
|
||||
## 2. Injest scraped urls into a sqlite database
|
||||
|
||||
This step converts the .json files into a sqlite database to allow scraping to be paused and restarted easily.
|
||||
|
||||
Copy 01-scraper-urls/output/artists-with-tabs/*.json into 02-injest-sqlite/input/
|
||||
|
||||
Run
|
||||
> npm install
|
||||
> node 01-injest-sqlite.js
|
||||
|
||||
Note: this script queues up the artist inserts into the database and then waits for the inserts to finish. Don't be surprised if it hangs for a few hours (it took 6 hours on my SSD).
|
||||
|
||||
This script will create a sqlite database, output/tabs-no-text.db
|
||||
|
||||
## 3. (Optional) Split the sqlite database for parallelized scraping
|
||||
|
||||
This step splits the generated sqlite database into multiple databases so that you can more easily use multiple machines to scrape the site.
|
||||
|
||||
Copy the tabs-no-text.db into 03-splitter-sqlite/
|
||||
|
||||
Determine the number of machines that you want to run the scraper on. Call this number N
|
||||
|
||||
Open up tabs-no-text.db
|
||||
> sqlite3 tabs-no-text.db
|
||||
|
||||
Create a view that separates the tabs into buckets
|
||||
> CREATE VIEW tabs_bucketed AS SELECT *, NTILE(N) OVER (ORDER BY rowid) AS bucket FROM tabs;
|
||||
|
||||
For each machine, i:
|
||||
|
||||
1) Create a new database
|
||||
> sqlite3 tabs-i-no-text.db
|
||||
|
||||
2) Attach to the base database
|
||||
> ATTACH 'tabs-no-text.db' AS db2;
|
||||
|
||||
3) Create a table with the rows from the machine's bucket
|
||||
> CREATE TABLE tabs AS SELECT * FROM db2.tabs_bucketed WHERE bucket=i;
|
||||
|
||||
Make sure to hold onto your tabs-no-text.db database for the merging process.
|
||||
|
||||
## 4. Scrape the tabs
|
||||
|
||||
For each machine, i
|
||||
|
||||
Copy the machine's tabs database into 04-scraper-tabs/input/tabs.db
|
||||
|
||||
Important: Make sure to **change the name** of the database file to **tabs.db** OR the file name in 01-scrape.js:100 - filename: './input/tabs-laptop.db'.
|
||||
|
||||
### 4.1 (Recommended) Index your database
|
||||
|
||||
It's highly recommended to index your database
|
||||
|
||||
Open up 04-scraper-tabs/input/
|
||||
|
||||
Run
|
||||
> sqlite3 tabs.db
|
||||
|
||||
Run the following SQL queries
|
||||
CREATE INDEX IF NOT EXISTS tabs_scrape_id_idx ON tabs (scrape_id);
|
||||
CREATE INDEX IF NOT EXISTS tabs_tab_url_idx ON tabs (tab_url);
|
||||
CREATE INDEX IF NOT EXISTS tabs_type_name_idx ON tabs (type_name);
|
||||
|
||||
### 4.2 (Optional) Optimize the tab scraper
|
||||
|
||||
Customize 01-scrape-tabs.js
|
||||
|
||||
Key Lines:
|
||||
|
||||
1) Line 192: let queue = new ConcurrentQueue(5);
|
||||
- - Increasing this value will increase the number of concurrent requests sent.
|
||||
- - Note: Higher concurrent request counts result in more agressive scrapes that may run more quickly but also get you kicked off more quickly
|
||||
|
||||
2) Line 214: await sleep(100);
|
||||
- - Increasing this value (in ms) will increase the delay between the first few concurrent requests. This staggers the requests, potentially reducing the chance you get kicked off.
|
||||
|
||||
3) Line 183: LIMIT 300
|
||||
- - Increasing the value in this line will increase the number of tabs scraped from the database at a time before sending a status update. Lower values will query the database more but give more frequent status updates. Higher values will take up more space in process memory and give less frequent status updates.
|
||||
- - If this value is set too low, removed tab urls will likely fill up the result set, causing the program to incorrectly detect that it got kicked off.
|
||||
|
||||
I found that running 500 tabs/minute gave me a good balance in effort spent reconnecting to VPN and scraping speed. Typically I would have to reset the scraper every 4-6 hours with this rate.
|
||||
|
||||
I got this with concurrency=5 and sleep=100.
|
||||
|
||||
### 4.4 Add required columns for scraping
|
||||
|
||||
Open your tabs.db in sqlite
|
||||
> sqlite3 tabs.db
|
||||
|
||||
Run the following commands to add the needed columns:
|
||||
|
||||
ALTER TABLE tabs DROP COLUMN tab_text;
|
||||
ALTER TABLE tabs ADD COLUMN user_id INTEGER;
|
||||
ALTER TABLE tabs ADD COLUMN user_iq INTEGER;
|
||||
ALTER TABLE tabs ADD COLUMN username TEXT;
|
||||
ALTER TABLE tabs ADD COLUMN tab_text TEXT;
|
||||
|
||||
### 4.3 Scrape the tabs
|
||||
|
||||
Important: Make sure to **change the name** of the database file to **tabs.db** OR the file name in 01-scrape.js:100 - filename: './input/tabs-laptop.db'.
|
||||
|
||||
Run
|
||||
> npm install
|
||||
> node 01-scrape-tabs.js
|
||||
|
||||
This will add tab information to the tabs.db database.
|
||||
|
||||
Note: This scraper only works for the following tab types:
|
||||
- Bass
|
||||
- Chords
|
||||
- Drums
|
||||
- Tab
|
||||
- Ukulele
|
||||
|
||||
The following tab types are not supported:
|
||||
- Guitar Pro
|
||||
- Official
|
||||
- Power
|
||||
- Video
|
||||
|
||||
## 5. (Optional) Merge the sqlite databases
|
||||
|
||||
Move the partial tabs databases from each machine to 05-merger-sqlite/input/{tabs-i.db}
|
||||
- Note: i is the machine number from before
|
||||
|
||||
Move the tabs-no-text.db database from step 2 into 05-merger-sqlite/input/.
|
||||
|
||||
Open up tabs-no-text.db
|
||||
> sqlite3 tabs-no-text.db
|
||||
|
||||
Create an index on tabs.tab_url
|
||||
> CREATE INDEX tabs_tab_url_idx ON tabs (tab_url);
|
||||
|
||||
Open up the final tabs database, tabs-full.db
|
||||
> sqlite3 tabs-full.db
|
||||
|
||||
Attach the no-text database
|
||||
> ATTACH 'tabs-no-text.db' AS 'dbnt';
|
||||
|
||||
Attach each machine database in the following format:
|
||||
> ATTACH 'tabs-i.db' AS 'dbi';
|
||||
|
||||
Create a the final tabs table
|
||||
CREATE TABLE tabs (
|
||||
scrape_id INTEGER
|
||||
, artist_scrape_id INTEGER NOT NULL
|
||||
, id INTEGER
|
||||
, song_id INTEGER
|
||||
, song_name TEXT
|
||||
, artist_id INTEGER
|
||||
, artist_name INTEGER
|
||||
, type TEXT
|
||||
, part TEXT
|
||||
, version INTEGER
|
||||
, votes INTEGER
|
||||
, rating NUMERIC
|
||||
, date TEXT
|
||||
, status TEXT
|
||||
, preset_id INTEGER
|
||||
, tab_access_type TEXT
|
||||
, tp_version INTEGER
|
||||
, tonality_name TEXT
|
||||
, version_description TEXT
|
||||
, verified INTEGER
|
||||
, artist_url TEXT
|
||||
, tab_url TEXT
|
||||
, difficulty TEXT
|
||||
, tuning TEXT
|
||||
, type_name TEXT
|
||||
, user_id INTEGER
|
||||
, user_iq INTEGER
|
||||
, username TEXT
|
||||
, tab_text TEXT
|
||||
);
|
||||
|
||||
For each machine, insert its respective tabs into the table
|
||||
INSERT INTO
|
||||
tabs
|
||||
SELECT
|
||||
tabsnt.scrape_id
|
||||
, tabsnt.artist_scrape_id
|
||||
, tabsnt.id
|
||||
, tabsnt.song_id
|
||||
, tabsnt.song_name
|
||||
, tabsnt.artist_id
|
||||
, tabsnt.artist_name
|
||||
, tabsnt.type
|
||||
, tabsnt.part
|
||||
, tabsnt.version
|
||||
, tabsnt.votes
|
||||
, tabsnt.rating
|
||||
, tabsnt.date
|
||||
, tabsnt.status
|
||||
, tabsnt.preset_id
|
||||
, tabsnt.tab_access_type
|
||||
, tabsnt.tp_version
|
||||
, tabsnt.tonality_name
|
||||
, tabsnt.version_description
|
||||
, tabsnt.verified
|
||||
, tabsnt.artist_url
|
||||
, tabsnt.tab_url
|
||||
, tabsnt.difficulty
|
||||
, tabsnt.tuning
|
||||
, tabsnt.type_name
|
||||
, tabsm.user_id
|
||||
, tabsm.user_iq
|
||||
, tabsm.username
|
||||
, tabsm.tab_text
|
||||
FROM
|
||||
dbnt.tabs AS tabsnt
|
||||
JOIN dbi.tabs AS tabsm ON tabsnt.tab_url=tabsm.tab_url
|
||||
WHERE
|
||||
tabsm.tab_url IS NOT NULL
|
||||
AND tabsm.tab_text IS NOT NULL;
|
||||
|
||||
Note: this command can take a bit to complete (30s-2m) depending on how large your databases are.
|
||||
|
||||
## 6. Print the contents of the database into organized text files
|
||||
|
||||
Copy your filled tabs database to 06-output-generator/input/tabs-full.db
|
||||
|
||||
Note: Make sure you either rename it in the directory or update 01-output-generator.js:52 with the proper file name
|
||||
|
||||
Run
|
||||
> npm install
|
||||
> node --max-old-space-size=16384 01-output-generator.js
|
||||
|
||||
Note: depending on how many tabs you scraped, you may have to increase the max-old-space-size (Max RAM). This example uses 16GB of ram.
|
||||
- I'm suspicious the memory leak is in the sqlite package >:|
|
||||
|
||||
Congratulations! Your guitar tabs are now organized in:
|
||||
06-output-generator/output/{type}/{artist}-{artist_id}/{song}.txt
|
||||
|
||||
## Other Information
|
||||
|
||||
You can customize the output generator's file output by modifying the fileText variable in 01-output-gernerator.js:84-99
|
||||
|
||||
Note: The .keep files can be ignored/deleted. They are kept to keep the default directory structure in git.
|
||||
Loading…
Reference in New Issue
Block a user