feat: initial commit

This commit is contained in:
Aaron Yarborough 2024-12-18 11:23:05 +00:00
commit 98f87f208d
15 changed files with 317402 additions and 0 deletions

130
.gitignore vendored Normal file
View file

@ -0,0 +1,130 @@
# Logs
logs
*.log
npm-debug.log*
yarn-debug.log*
yarn-error.log*
lerna-debug.log*
.pnpm-debug.log*
# Diagnostic reports (https://nodejs.org/api/report.html)
report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json
# Runtime data
pids
*.pid
*.seed
*.pid.lock
# Directory for instrumented libs generated by jscoverage/JSCover
lib-cov
# Coverage directory used by tools like istanbul
coverage
*.lcov
# nyc test coverage
.nyc_output
# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files)
.grunt
# Bower dependency directory (https://bower.io/)
bower_components
# node-waf configuration
.lock-wscript
# Compiled binary addons (https://nodejs.org/api/addons.html)
build/Release
# Dependency directories
node_modules/
jspm_packages/
# Snowpack dependency directory (https://snowpack.dev/)
web_modules/
# TypeScript cache
*.tsbuildinfo
# Optional npm cache directory
.npm
# Optional eslint cache
.eslintcache
# Optional stylelint cache
.stylelintcache
# Microbundle cache
.rpt2_cache/
.rts2_cache_cjs/
.rts2_cache_es/
.rts2_cache_umd/
# Optional REPL history
.node_repl_history
# Output of 'npm pack'
*.tgz
# Yarn Integrity file
.yarn-integrity
# dotenv environment variable files
.env
.env.development.local
.env.test.local
.env.production.local
.env.local
# parcel-bundler cache (https://parceljs.org/)
.cache
.parcel-cache
# Next.js build output
.next
out
# Nuxt.js build / generate output
.nuxt
dist
# Gatsby files
.cache/
# Comment in the public line in if your project uses Gatsby and not Next.js
# https://nextjs.org/blog/next-9-1#public-directory-support
# public
# vuepress build output
.vuepress/dist
# vuepress v2.x temp and cache directory
.temp
.cache
# Docusaurus cache and generated files
.docusaurus
# Serverless directories
.serverless/
# FuseBox cache
.fusebox/
# DynamoDB Local files
.dynamodb/
# TernJS port file
.tern-port
# Stores VSCode versions used for testing VSCode extensions
.vscode-test
# yarn v2
.yarn/cache
.yarn/unplugged
.yarn/build-state.yml
.yarn/install-state.gz
.pnp.*

1
.nvmrc Normal file
View file

@ -0,0 +1 @@
v23.4.0

6
.vscode/settings.json vendored Normal file
View file

@ -0,0 +1,6 @@
{
"cSpell.words": [
"insomnium",
"REQS"
]
}

0
data/.gitkeep Normal file
View file

27622
data/Bristol Council.json Normal file

File diff suppressed because one or more lines are too long

View file

@ -0,0 +1 @@
[]

File diff suppressed because one or more lines are too long

File diff suppressed because it is too large Load diff

183109
data/Southampton Council.json Normal file

File diff suppressed because one or more lines are too long

7
helpers.js Normal file
View file

@ -0,0 +1,7 @@
function wait(ms) {
return new Promise(resolve => setTimeout(resolve, ms));
}
module.exports = {
wait
};

318
insomnium.json Normal file
View file

@ -0,0 +1,318 @@
{
"_type": "export",
"__export_format": 4,
"__export_date": "2024-12-13T14:50:44.419Z",
"__export_source": "insomnia.desktop.app:v0.2.2",
"resources": [
{
"_id": "req_5a17e696d356437d8b08f73db7d53d9f",
"parentId": "fld_7249563264484823be781bbc7ce7d972",
"modified": 1734101410939,
"created": 1734101404967,
"url": "https://directory.southampton.gov.uk/api/services",
"name": "/services",
"description": "",
"method": "GET",
"body": {},
"parameters": [],
"headers": [
{
"name": "User-Agent",
"value": "insomnia/0.2.2"
}
],
"authentication": {},
"metaSortKey": -1734101404967,
"isPrivate": false,
"settingStoreCookies": true,
"settingSendCookies": true,
"settingDisableRenderRequestBody": false,
"settingEncodeUrl": true,
"settingRebuildPath": true,
"settingFollowRedirects": "global",
"segmentParams": [],
"_type": "request"
},
{
"_id": "fld_7249563264484823be781bbc7ce7d972",
"parentId": "wrk_b6699c43c9a1492787836ea5019c5982",
"modified": 1734101402543,
"created": 1734101402543,
"name": "Southampton Council",
"description": "",
"environment": {},
"environmentPropertyOrder": null,
"metaSortKey": -1734101402543,
"_type": "request_group"
},
{
"_id": "wrk_b6699c43c9a1492787836ea5019c5982",
"parentId": null,
"modified": 1734101147095,
"created": 1734101147095,
"name": "ORUK LAs",
"description": "",
"scope": "collection",
"_type": "workspace"
},
{
"_id": "req_e2cc674b1e1b41bd8e4b95b78c65c7f8",
"parentId": "fld_12fd3f34f0ea4844bba2909a227cdd1a",
"modified": 1734101381629,
"created": 1734101358188,
"url": "https://penninelancs.openplace.directory/o/ServiceDirectoryService/v2/services",
"name": "/services",
"description": "",
"method": "GET",
"body": {},
"parameters": [],
"headers": [
{
"name": "User-Agent",
"value": "insomnia/0.2.2"
}
],
"authentication": {},
"metaSortKey": -1734101358188,
"isPrivate": false,
"settingStoreCookies": true,
"settingSendCookies": true,
"settingDisableRenderRequestBody": false,
"settingEncodeUrl": true,
"settingRebuildPath": true,
"settingFollowRedirects": "global",
"segmentParams": [],
"_type": "request"
},
{
"_id": "fld_12fd3f34f0ea4844bba2909a227cdd1a",
"parentId": "wrk_b6699c43c9a1492787836ea5019c5982",
"modified": 1734101355947,
"created": 1734101355947,
"name": "Pennine Lancashire ICP",
"description": "",
"environment": {},
"environmentPropertyOrder": null,
"metaSortKey": -1734101355947,
"_type": "request_group"
},
{
"_id": "req_dc6bb46f106f43e79ce5e0d7dc9051a5",
"parentId": "fld_0db8e7de61bc4d11a07117caea843b44",
"modified": 1734101326671,
"created": 1734101319555,
"url": "https://northlincs.openplace.directory/o/ServiceDirectoryService/v2/services",
"name": "/services",
"description": "",
"method": "GET",
"body": {},
"parameters": [],
"headers": [
{
"name": "User-Agent",
"value": "insomnia/0.2.2"
}
],
"authentication": {},
"metaSortKey": -1734101319555,
"isPrivate": false,
"settingStoreCookies": true,
"settingSendCookies": true,
"settingDisableRenderRequestBody": false,
"settingEncodeUrl": true,
"settingRebuildPath": true,
"settingFollowRedirects": "global",
"segmentParams": [],
"_type": "request"
},
{
"_id": "fld_0db8e7de61bc4d11a07117caea843b44",
"parentId": "wrk_b6699c43c9a1492787836ea5019c5982",
"modified": 1734101317039,
"created": 1734101317039,
"name": "North Lincolnshire Council",
"description": "",
"environment": {},
"environmentPropertyOrder": null,
"metaSortKey": -1734101317039,
"_type": "request_group"
},
{
"_id": "req_35869d2eaffc4819858002ce7d5131f6",
"parentId": "fld_7db16d60402144d2ba30852b58a5bd82",
"modified": 1734101288057,
"created": 1734101280037,
"url": "https://api.familyinfo.buckinghamshire.gov.uk/api/v1/services",
"name": "/services",
"description": "",
"method": "GET",
"body": {},
"parameters": [],
"headers": [
{
"name": "User-Agent",
"value": "insomnia/0.2.2"
}
],
"authentication": {},
"metaSortKey": -1734101284292,
"isPrivate": false,
"settingStoreCookies": true,
"settingSendCookies": true,
"settingDisableRenderRequestBody": false,
"settingEncodeUrl": true,
"settingRebuildPath": true,
"settingFollowRedirects": "global",
"segmentParams": [],
"_type": "request"
},
{
"_id": "fld_7db16d60402144d2ba30852b58a5bd82",
"parentId": "wrk_b6699c43c9a1492787836ea5019c5982",
"modified": 1734101277775,
"created": 1734101277775,
"name": "Buckinghamshire Council",
"description": "",
"environment": {},
"environmentPropertyOrder": null,
"metaSortKey": -1734101277775,
"_type": "request_group"
},
{
"_id": "req_c2e99c36c4ec4deb9c283e6a47076f3d",
"parentId": "fld_ab7b6940dd0c44049d4e41af5c68637c",
"modified": 1734101205414,
"created": 1734101149071,
"url": "https://bristol.openplace.directory/o/ServiceDirectoryService/v2/hservices",
"name": "/services",
"description": "",
"method": "GET",
"body": {},
"parameters": [],
"headers": [
{
"name": "User-Agent",
"value": "insomnia/0.2.2"
}
],
"authentication": {},
"metaSortKey": -1734101205391,
"isPrivate": false,
"settingStoreCookies": true,
"settingSendCookies": true,
"settingDisableRenderRequestBody": false,
"settingEncodeUrl": true,
"settingRebuildPath": true,
"settingFollowRedirects": "global",
"segmentParams": [],
"_type": "request"
},
{
"_id": "fld_ab7b6940dd0c44049d4e41af5c68637c",
"parentId": "wrk_b6699c43c9a1492787836ea5019c5982",
"modified": 1734101191560,
"created": 1734101191560,
"name": "Bristol Council",
"description": "",
"environment": {},
"environmentPropertyOrder": null,
"metaSortKey": -1734101191560,
"_type": "request_group"
},
{
"_id": "env_de6c835521010ba26be4220f11dc5a6f5069ddf3",
"parentId": "wrk_b6699c43c9a1492787836ea5019c5982",
"modified": 1734101147098,
"created": 1734101147098,
"name": "Base Environment",
"data": {},
"dataPropertyOrder": null,
"color": null,
"isPrivate": false,
"metaSortKey": 1734101147098,
"_type": "environment"
},
{
"_id": "jar_de6c835521010ba26be4220f11dc5a6f5069ddf3",
"parentId": "wrk_b6699c43c9a1492787836ea5019c5982",
"modified": 1734101376960,
"created": 1734101147098,
"name": "Default Jar",
"cookies": [
{
"key": "JSESSIONID",
"value": "1B079D52EEAE9DBC6783C96D71EDF399",
"domain": "bristol.openplace.directory",
"path": "/",
"secure": true,
"httpOnly": true,
"hostOnly": true,
"creation": "2024-12-13T14:46:10.292Z",
"lastAccessed": "2024-12-13T14:46:10.292Z",
"id": "4630461386775335"
},
{
"key": "AWSELB",
"value": "2D870D5F0E8860F2EF95DB74F9FDC976417AF8CE68BB383FC3865E321F4838F1CD6B5C9AF23B387FD2E2AAA3FA0C38F5BEF0D9172D5C9046BC720034DB7DD4D5154F9B072EE2AA12398CEDCEC6C4A209C1F79F782A",
"domain": "bristol.openplace.directory",
"path": "/",
"secure": true,
"httpOnly": true,
"hostOnly": true,
"creation": "2024-12-13T14:46:10.294Z",
"lastAccessed": "2024-12-13T14:46:10.294Z",
"id": "12139281671488988"
},
{
"key": "JSESSIONID",
"value": "DF04FB103146B7994E88BCC9E7B72F31",
"domain": "northlincs.openplace.directory",
"path": "/",
"secure": true,
"httpOnly": true,
"hostOnly": true,
"creation": "2024-12-13T14:48:41.552Z",
"lastAccessed": "2024-12-13T14:48:41.552Z",
"id": "1023352684773664"
},
{
"key": "AWSELB",
"value": "2D870D5F0E8860F2EF95DB74F9FDC976417AF8CE6809D3AEDDCA61FB82F1F2060AEEA8F11C6541C0E2BA34496CB5FB94A03D9314795C9046BC720034DB7DD4D5154F9B072EE2AA12398CEDCEC6C4A209C1F79F782A",
"domain": "northlincs.openplace.directory",
"path": "/",
"secure": true,
"httpOnly": true,
"hostOnly": true,
"creation": "2024-12-13T14:48:41.552Z",
"lastAccessed": "2024-12-13T14:48:41.552Z",
"id": "8515615299671728"
},
{
"key": "JSESSIONID",
"value": "0144C070FC1D17CE851A8B6A2C8F5B61",
"domain": "penninelancs.openplace.directory",
"path": "/",
"secure": true,
"httpOnly": true,
"hostOnly": true,
"creation": "2024-12-13T14:49:36.958Z",
"lastAccessed": "2024-12-13T14:49:36.958Z",
"id": "7743775872235581"
},
{
"key": "AWSELB",
"value": "2D870D5F0E8860F2EF95DB74F9FDC976417AF8CE68BFA6CBA6790DEEA2DD2A9AC9FAFF5B7776B09675239D14A769B864D1960639ED5C9046BC720034DB7DD4D5154F9B072EE2AA12398CEDCEC6C4A209C1F79F782A",
"domain": "penninelancs.openplace.directory",
"path": "/",
"secure": true,
"httpOnly": true,
"hostOnly": true,
"creation": "2024-12-13T14:49:36.959Z",
"lastAccessed": "2024-12-13T14:49:36.959Z",
"id": "8352805125006024"
}
],
"_type": "cookie_jar"
}
]
}

26
package-lock.json generated Normal file
View file

@ -0,0 +1,26 @@
{
"name": "fh-or-tester",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "fh-or-tester",
"dependencies": {
"uuid": "^11.0.3"
}
},
"node_modules/uuid": {
"version": "11.0.3",
"resolved": "https://registry.npmjs.org/uuid/-/uuid-11.0.3.tgz",
"integrity": "sha512-d0z310fCWv5dJwnX1Y/MncBAqGMKEzlBb1AOf7z9K8ALnd0utBX/msg/fA0+sbyN1ihbMsLhrBlnl1ak7Wa0rg==",
"funding": [
"https://github.com/sponsors/broofa",
"https://github.com/sponsors/ctavan"
],
"license": "MIT",
"bin": {
"uuid": "dist/esm/bin/uuid"
}
}
}
}

11
package.json Normal file
View file

@ -0,0 +1,11 @@
{
"name": "fh-or-tester",
"main": "index.js",
"scripts": {
"fetch": "node script-fetch.js",
"eval": "node script-eval.js"
},
"dependencies": {
"uuid": "^11.0.3"
}
}

74
script-eval.js Normal file
View file

@ -0,0 +1,74 @@
const uuid = require("uuid");
const fs = require('fs');
(async () => {
const path = process.argv[0];
if (!path) {
throw new Error("Path not provided.");
}
const results = [];
const services = await loadData(process.argv[2]);
for (let service of services) {
results.push({
id: service.id,
hasValidID: hasValidID(service),
hasValidName: hasValidName(service),
hasValidDescription: hasValidDescription(service),
hasValidURL: hasValidURL(service),
hasValidOrganisation: hasValidOrganisation(service),
hasValidContact: hasValidContact(service)
});
}
console.table(results);
const totalWithValidID = results.filter(x => x.hasValidID).length;
console.log(`# of services with a valid ID: ${totalWithValidID}/${results.length} (${Math.round((totalWithValidID/results.length)*100)}%)`);
const totalWithValidName = results.filter(x => x.hasValidName).length;
console.log(`# of services with a valid name: ${totalWithValidName}/${results.length} (${Math.round((totalWithValidName/results.length)*100)}%)`);
const totalWithValidDescription = results.filter(x => x.hasValidDescription).length;
console.log(`# of services with a valid description: ${totalWithValidDescription}/${results.length} (${Math.round((totalWithValidDescription/results.length)*100)}%)`);
const totalWithValidURL = results.filter(x => x.hasValidURL).length;
console.log(`# of services with a valid URL: ${totalWithValidURL}/${results.length} (${Math.round((totalWithValidURL/results.length)*100)}%)`);
const totalWithValidOrganisation = results.filter(x => x.hasValidOrganisation).length;
console.log(`# of services with a valid organisation: ${totalWithValidOrganisation}/${results.length} (${Math.round((totalWithValidOrganisation/results.length)*100)}%)`);
const totalWithValidContact = results.filter(x => x.hasValidContact).length;
console.log(`# of services with a valid contact: ${totalWithValidContact}/${results.length} (${Math.round((totalWithValidContact/results.length)*100)}%)`);
})();
async function loadData(path) {
console.log("Loading from " + path);
const text = await fs.promises.readFile(path, {
encoding: "utf-8"
});
return JSON.parse(text);
}
// Validators
const hasValidID = (s) => uuid.validate(s.id);
const hasValidName = (s) => typeof s.name === "string" && s.name.length > 0;
const hasValidDescription = (s) => typeof s.description === "string" && s.description.length > 0;
const hasValidURL = (s) => {
try {
new URL(s.url);
return true;
} catch (_) {
return false;
}
};
const hasValidOrganisation = (s) => !!s.organization?.id && !!s.organization?.name;
const hasValidContact = (s) => !!s.email?.length > 0;

68
script-fetch.js Normal file
View file

@ -0,0 +1,68 @@
const { wait } = require("./helpers");
const insomnium = require("./insomnium.json");
const fs = require("fs");
const WAIT_BETWEEN_REQS_MS = 500;
const PER_PAGE=500;
const groups = insomnium.resources
.filter(res => res._type === "request_group");
const las = insomnium.resources
.filter(res =>
res._type === "request" &&
res.name === "/services")
.map(res => ({
group: groups.find(group => group._id == res.parentId),
res
}));
(async() => {
console.log('Fetching LAs...');
for (let la of las) {
const laName = la.group.name;
console.log(`Fetching services for ${laName}...`);
const services = await fetchServices(la.res.url);
console.log(`Found ${services.length} services for ${laName}!`);
await writeServices(laName, services);
}
})();
async function fetchServices(ep) {
let page = 1;
let services = [];
do {
const pagedEp = `${ep}?page=${page}&per_page=${PER_PAGE}`;
const resp = await fetch(pagedEp);
console.log(pagedEp);
if (!resp.ok) {
console.warn(`Could not fetch services for ${ep}. Skipping. (Resp code ${resp.status})`);
return [];
}
const json = await resp.json();
services = [...services, ...json.content];
if (json.last == true) {
console.log("All services found!")
break;
}
console.log(`Loaded ${services.length} of total ${json.totalElements}...`);
page++;
await wait(WAIT_BETWEEN_REQS_MS);
} while (true);
return services;
}
async function writeServices(name, services) {
await fs.promises.writeFile(`data/${name}.json`, JSON.stringify(services, null, 2));
}