Compare commits
1 commit
| Author | SHA1 | Date | |
|---|---|---|---|
| a9f54481f3 |
16 changed files with 317291 additions and 275 deletions
4
.gitignore
vendored
4
.gitignore
vendored
|
|
@ -127,6 +127,4 @@ dist
|
|||
.yarn/unplugged
|
||||
.yarn/build-state.yml
|
||||
.yarn/install-state.gz
|
||||
.pnp.*
|
||||
|
||||
data/*
|
||||
.pnp.*
|
||||
52
README.md
52
README.md
|
|
@ -6,59 +6,33 @@ A tool to fetch and validate the quality of data from Open Referral UK APIs.
|
|||
|
||||
The Family Hubs DfE delivery team use this tool to pull and quality check the data available on current (as of Dec 2024) Open Referral UK APIs.
|
||||
|
||||
## Pre-requisites
|
||||
## Data source
|
||||
|
||||
- node (built using v23.4.0; will probably run on nearby versions)
|
||||
The list of APIs was gathered from here: https://openreferraluk.org/dashboard
|
||||
|
||||
## Usage
|
||||
|
||||
1. Clone the repo
|
||||
2. Run `npm install` to install node dependencies
|
||||
3. Run `node bin/fetch.js <url>` to fetch and save a JSON file containing an array of services data.
|
||||
4. Run `node bin/eval.js <path>` to run the evaluation tool over a JSON file generated from step 3.
|
||||
3. Run `npm run fetch` to fetch up to date data
|
||||
4. Run `npm run eval ./data/Bristol\ Council.json` to run the evaluation tool over the pulled data (swapping the path to whatever LA data you want to evaluate)
|
||||
|
||||
Results will be shown in the console:
|
||||
|
||||
```sh
|
||||
```
|
||||
...
|
||||
│ 817 │ 'f92ddae3-2007-8cf8-c9eb-ec2aad962f2a' │ false │ true │ true │ true │ true │ true │
|
||||
│ 818 │ 'dbb17998-b11a-45e7-ac1f-2bae60e40e0a' │ true │ true │ true │ true │ true │ false │
|
||||
└─────────┴────────────────────────────────────────┴────────────┴──────────────┴─────────────────────┴─────────────┴──────────────────────┴─────────────────┘
|
||||
--------------------
|
||||
RESULTS
|
||||
--------------------
|
||||
# of services with a valid ID: 0/1157 (0%)
|
||||
# of services with a valid status: 1152/1157 (100%)
|
||||
# of services with a valid name: 1157/1157 (100%)
|
||||
# of services with a valid description: 1135/1157 (98%)
|
||||
# of services with a valid URL: 979/1157 (85%)
|
||||
# of services with a valid organisation: 1149/1157 (99%)
|
||||
# of services with a valid contact: 0/1157 (0%)
|
||||
--------------------
|
||||
# of usable services: 0/1157 (0%)
|
||||
--------------------
|
||||
# of services with a valid ID: 399/819 (49%)
|
||||
# of services with a valid name: 818/819 (100%)
|
||||
# of services with a valid description: 819/819 (100%)
|
||||
# of services with a valid URL: 691/819 (84%)
|
||||
# of services with a valid organisation: 812/819 (99%)
|
||||
# of services with a valid contact: 598/819 (73%)
|
||||
```
|
||||
|
||||
**Note:** use the `--help` CLI flag to see more options for each command.
|
||||
### Updating the list of APIs
|
||||
|
||||
### Example usage
|
||||
|
||||
To fetch and evaluate the quality of data from Southampton's OR UK API (https://directory.southampton.gov.uk/api):
|
||||
|
||||
```sh
|
||||
$ node bin/fetch.js https://directory.southampton.gov.uk/api/services
|
||||
$ node bin/eval.js ./data/directorysouthamptongovuk.json
|
||||
```
|
||||
|
||||
### Fetching all current OR UK-compliant local authority data
|
||||
|
||||
The following snippet will fetch data from all current (as of Dec 2024) OR UK-compliant local authorities with APIs on the [OR UK dashboard](https://openreferraluk.org/dashboard):
|
||||
|
||||
```sh
|
||||
node bin/fetch.js https://bristol.openplace.directory/o/ServiceDirectoryService/v2/services
|
||||
node bin/fetch.js https://northlincs.openplace.directory/o/ServiceDirectoryService/v2/services
|
||||
node bin/fetch.js https://directory.southampton.gov.uk/api/services
|
||||
node bin/fetch.js https://api.familyinfo.buckinghamshire.gov.uk/api/v1/services
|
||||
node bin/fetch.js https://penninelancs.openplace.directory/o/ServiceDirectoryService/v2/services
|
||||
```
|
||||
An insomnium JSON file (`./insomoum.json`) is used to store the list of endpoints the tool fetches data from. To edit the list using a UI, download and install the [Insomnium](https://github.com/ArchGPT/insomnium) REST API client and open the JSON file.
|
||||
|
||||
|
|
|
|||
109
bin/eval.js
109
bin/eval.js
|
|
@ -1,109 +0,0 @@
|
|||
const fs = require('fs');
|
||||
const { hasValidId, hasValidStatus, hasValidName, hasValidDescription, hasValidOrganisation, hasValidContact, hasValidUrl } = require('../validators');
|
||||
const { program } = require("commander");
|
||||
|
||||
program
|
||||
.argument("<path>", "Path to the JSON data file.")
|
||||
.option("-s, --show-services", "Shows detailed validation information for each service in a table.", false);
|
||||
|
||||
program.parse();
|
||||
|
||||
const options = program.opts();
|
||||
|
||||
(async () => {
|
||||
const path = process.argv[2];
|
||||
if (!path) {
|
||||
console.error("Path not provided.");
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
// Load services array from the given JSON file
|
||||
const services = await loadData(path);
|
||||
// Iterate over and validate all services
|
||||
const results = services.map(service => ({
|
||||
id: service.id,
|
||||
hasValidID: hasValidId(service),
|
||||
hasValidStatus: hasValidStatus(service),
|
||||
hasValidName: hasValidName(service),
|
||||
hasValidDescription: hasValidDescription(service),
|
||||
hasValidURL: hasValidUrl(service),
|
||||
hasValidOrganisation: hasValidOrganisation(service),
|
||||
hasValidContact: hasValidContact(service)
|
||||
}));
|
||||
|
||||
if (options.showServices) {
|
||||
console.table(results);
|
||||
}
|
||||
|
||||
const validationResultMessage = (text, validTotal) =>
|
||||
`${text}: ${validTotal}/${results.length} (${Math.round((validTotal / results.length) * 100)}%)`;
|
||||
|
||||
console.log("--------------------");
|
||||
console.log("RESULTS");
|
||||
console.log("--------------------");
|
||||
|
||||
console.log(validationResultMessage(
|
||||
"# of services with a valid ID",
|
||||
results.filter(x => x.hasValidID).length
|
||||
));
|
||||
|
||||
console.log(validationResultMessage(
|
||||
"# of services with a valid status",
|
||||
results.filter(x => x.hasValidStatus).length,
|
||||
));
|
||||
|
||||
console.log(validationResultMessage(
|
||||
"# of services with a valid name",
|
||||
results.filter(x => x.hasValidName).length
|
||||
));
|
||||
|
||||
console.log(validationResultMessage(
|
||||
"# of services with a valid description",
|
||||
results.filter(x => x.hasValidDescription).length
|
||||
));
|
||||
|
||||
console.log(validationResultMessage(
|
||||
"# of services with a valid URL",
|
||||
results.filter(x => x.hasValidURL).length
|
||||
));
|
||||
|
||||
console.log(validationResultMessage(
|
||||
"# of services with a valid organisation",
|
||||
results.filter(x => x.hasValidOrganisation).length
|
||||
));
|
||||
|
||||
console.log(validationResultMessage(
|
||||
"# of services with a valid contact",
|
||||
results.filter(x => x.hasValidContact).length
|
||||
));
|
||||
|
||||
console.log("--------------------");
|
||||
|
||||
console.log(validationResultMessage(
|
||||
"# of usable services",
|
||||
results.filter(x =>
|
||||
x.hasValidID &&
|
||||
x.hasValidStatus &&
|
||||
x.hasValidName &&
|
||||
x.hasValidDescription &&
|
||||
x.hasValidURL &&
|
||||
x.hasValidOrganisation &&
|
||||
x.hasValidContact).length));
|
||||
|
||||
console.log("--------------------");
|
||||
|
||||
})();
|
||||
|
||||
async function loadData(path) {
|
||||
console.log(`Loading JSON data from ${path}...`);
|
||||
|
||||
try {
|
||||
const text = await fs.promises.readFile(path, {
|
||||
encoding: "utf-8"
|
||||
});
|
||||
|
||||
return JSON.parse(text);
|
||||
} catch (error) {
|
||||
throw new Error("Unable to load JSON data file", { cause: error });
|
||||
}
|
||||
}
|
||||
63
bin/fetch.js
63
bin/fetch.js
|
|
@ -1,63 +0,0 @@
|
|||
const { wait, toFriendlyFilename } = require("../helpers");
|
||||
const fs = require("fs");
|
||||
const { program } = require("commander");
|
||||
|
||||
program
|
||||
.argument("<url>", "Services URL of the OR UK API to fetch data from.")
|
||||
.option("-o, --out-path", "Output path to save fetched JSON data to.", "./data/<url>.json")
|
||||
.option("-w, --wait-between-req-ms", "The number of milliseconds to wait between API requests.", 500)
|
||||
.option("-p, --per-page", "The number of items to return in each response.", 500);
|
||||
|
||||
program.parse();
|
||||
|
||||
const options = program.opts();
|
||||
|
||||
(async () => {
|
||||
const url = program.args[0];
|
||||
const host = new URL(url).host;
|
||||
const outPath = (options.outPath.replace("<url>", toFriendlyFilename(host)));
|
||||
|
||||
// Get services
|
||||
console.log(`Fetching services from ${url}...`);
|
||||
const services = await fetchServices(url, options.perPage, options.waitBetweenReqMs);
|
||||
|
||||
// Write to file
|
||||
await fs.promises.writeFile(
|
||||
outPath,
|
||||
JSON.stringify(services, null, 2)
|
||||
);
|
||||
console.log(`Wrote services ${services.length} to ${outPath}.`)
|
||||
})();
|
||||
|
||||
async function fetchServices(ep, perPage, waitMs) {
|
||||
let page = 1;
|
||||
let services = [];
|
||||
|
||||
do {
|
||||
const pagedEp = `${ep}?page=${page}&per_page=${perPage}`;
|
||||
const resp = await fetch(pagedEp);
|
||||
|
||||
if (!resp.ok) {
|
||||
console.warn(
|
||||
`Could not fetch services for ${ep}. Skipping. (Resp code ${resp.status})`
|
||||
);
|
||||
return [];
|
||||
}
|
||||
|
||||
const json = await resp.json();
|
||||
|
||||
services = [...services, ...json.content];
|
||||
|
||||
if (json.last == true) {
|
||||
console.log("All services found!");
|
||||
break;
|
||||
}
|
||||
|
||||
console.log(`Loaded ${services.length} of total ${json.totalElements}...`);
|
||||
|
||||
page++;
|
||||
await wait(waitMs);
|
||||
} while (true);
|
||||
|
||||
return services;
|
||||
}
|
||||
27622
data/Bristol Council.json
Normal file
27622
data/Bristol Council.json
Normal file
File diff suppressed because one or more lines are too long
1
data/Buckinghamshire Council.json
Normal file
1
data/Buckinghamshire Council.json
Normal file
|
|
@ -0,0 +1 @@
|
|||
[]
|
||||
24389
data/North Lincolnshire Council.json
Normal file
24389
data/North Lincolnshire Council.json
Normal file
File diff suppressed because one or more lines are too long
81640
data/Pennine Lancashire ICP.json
Normal file
81640
data/Pennine Lancashire ICP.json
Normal file
File diff suppressed because it is too large
Load diff
183109
data/Southampton Council.json
Normal file
183109
data/Southampton Council.json
Normal file
File diff suppressed because one or more lines are too long
11
helpers.js
11
helpers.js
|
|
@ -2,15 +2,6 @@ function wait(ms) {
|
|||
return new Promise(resolve => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
function toFriendlyFilename(str) {
|
||||
return str
|
||||
.trim() // Remove leading and trailing whitespace
|
||||
.toLowerCase() // Convert to lowercase
|
||||
.replace(/[^a-z0-9\s]/g, '') // Remove non-alphanumeric characters except spaces
|
||||
.replace(/\s+/g, '_'); // Replace spaces with underscores
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
wait,
|
||||
toFriendlyFilename
|
||||
wait
|
||||
};
|
||||
318
insomnium.json
Normal file
318
insomnium.json
Normal file
|
|
@ -0,0 +1,318 @@
|
|||
{
|
||||
"_type": "export",
|
||||
"__export_format": 4,
|
||||
"__export_date": "2024-12-13T14:50:44.419Z",
|
||||
"__export_source": "insomnia.desktop.app:v0.2.2",
|
||||
"resources": [
|
||||
{
|
||||
"_id": "req_5a17e696d356437d8b08f73db7d53d9f",
|
||||
"parentId": "fld_7249563264484823be781bbc7ce7d972",
|
||||
"modified": 1734101410939,
|
||||
"created": 1734101404967,
|
||||
"url": "https://directory.southampton.gov.uk/api/services",
|
||||
"name": "/services",
|
||||
"description": "",
|
||||
"method": "GET",
|
||||
"body": {},
|
||||
"parameters": [],
|
||||
"headers": [
|
||||
{
|
||||
"name": "User-Agent",
|
||||
"value": "insomnia/0.2.2"
|
||||
}
|
||||
],
|
||||
"authentication": {},
|
||||
"metaSortKey": -1734101404967,
|
||||
"isPrivate": false,
|
||||
"settingStoreCookies": true,
|
||||
"settingSendCookies": true,
|
||||
"settingDisableRenderRequestBody": false,
|
||||
"settingEncodeUrl": true,
|
||||
"settingRebuildPath": true,
|
||||
"settingFollowRedirects": "global",
|
||||
"segmentParams": [],
|
||||
"_type": "request"
|
||||
},
|
||||
{
|
||||
"_id": "fld_7249563264484823be781bbc7ce7d972",
|
||||
"parentId": "wrk_b6699c43c9a1492787836ea5019c5982",
|
||||
"modified": 1734101402543,
|
||||
"created": 1734101402543,
|
||||
"name": "Southampton Council",
|
||||
"description": "",
|
||||
"environment": {},
|
||||
"environmentPropertyOrder": null,
|
||||
"metaSortKey": -1734101402543,
|
||||
"_type": "request_group"
|
||||
},
|
||||
{
|
||||
"_id": "wrk_b6699c43c9a1492787836ea5019c5982",
|
||||
"parentId": null,
|
||||
"modified": 1734101147095,
|
||||
"created": 1734101147095,
|
||||
"name": "ORUK LAs",
|
||||
"description": "",
|
||||
"scope": "collection",
|
||||
"_type": "workspace"
|
||||
},
|
||||
{
|
||||
"_id": "req_e2cc674b1e1b41bd8e4b95b78c65c7f8",
|
||||
"parentId": "fld_12fd3f34f0ea4844bba2909a227cdd1a",
|
||||
"modified": 1734101381629,
|
||||
"created": 1734101358188,
|
||||
"url": "https://penninelancs.openplace.directory/o/ServiceDirectoryService/v2/services",
|
||||
"name": "/services",
|
||||
"description": "",
|
||||
"method": "GET",
|
||||
"body": {},
|
||||
"parameters": [],
|
||||
"headers": [
|
||||
{
|
||||
"name": "User-Agent",
|
||||
"value": "insomnia/0.2.2"
|
||||
}
|
||||
],
|
||||
"authentication": {},
|
||||
"metaSortKey": -1734101358188,
|
||||
"isPrivate": false,
|
||||
"settingStoreCookies": true,
|
||||
"settingSendCookies": true,
|
||||
"settingDisableRenderRequestBody": false,
|
||||
"settingEncodeUrl": true,
|
||||
"settingRebuildPath": true,
|
||||
"settingFollowRedirects": "global",
|
||||
"segmentParams": [],
|
||||
"_type": "request"
|
||||
},
|
||||
{
|
||||
"_id": "fld_12fd3f34f0ea4844bba2909a227cdd1a",
|
||||
"parentId": "wrk_b6699c43c9a1492787836ea5019c5982",
|
||||
"modified": 1734101355947,
|
||||
"created": 1734101355947,
|
||||
"name": "Pennine Lancashire ICP",
|
||||
"description": "",
|
||||
"environment": {},
|
||||
"environmentPropertyOrder": null,
|
||||
"metaSortKey": -1734101355947,
|
||||
"_type": "request_group"
|
||||
},
|
||||
{
|
||||
"_id": "req_dc6bb46f106f43e79ce5e0d7dc9051a5",
|
||||
"parentId": "fld_0db8e7de61bc4d11a07117caea843b44",
|
||||
"modified": 1734101326671,
|
||||
"created": 1734101319555,
|
||||
"url": "https://northlincs.openplace.directory/o/ServiceDirectoryService/v2/services",
|
||||
"name": "/services",
|
||||
"description": "",
|
||||
"method": "GET",
|
||||
"body": {},
|
||||
"parameters": [],
|
||||
"headers": [
|
||||
{
|
||||
"name": "User-Agent",
|
||||
"value": "insomnia/0.2.2"
|
||||
}
|
||||
],
|
||||
"authentication": {},
|
||||
"metaSortKey": -1734101319555,
|
||||
"isPrivate": false,
|
||||
"settingStoreCookies": true,
|
||||
"settingSendCookies": true,
|
||||
"settingDisableRenderRequestBody": false,
|
||||
"settingEncodeUrl": true,
|
||||
"settingRebuildPath": true,
|
||||
"settingFollowRedirects": "global",
|
||||
"segmentParams": [],
|
||||
"_type": "request"
|
||||
},
|
||||
{
|
||||
"_id": "fld_0db8e7de61bc4d11a07117caea843b44",
|
||||
"parentId": "wrk_b6699c43c9a1492787836ea5019c5982",
|
||||
"modified": 1734101317039,
|
||||
"created": 1734101317039,
|
||||
"name": "North Lincolnshire Council",
|
||||
"description": "",
|
||||
"environment": {},
|
||||
"environmentPropertyOrder": null,
|
||||
"metaSortKey": -1734101317039,
|
||||
"_type": "request_group"
|
||||
},
|
||||
{
|
||||
"_id": "req_35869d2eaffc4819858002ce7d5131f6",
|
||||
"parentId": "fld_7db16d60402144d2ba30852b58a5bd82",
|
||||
"modified": 1734101288057,
|
||||
"created": 1734101280037,
|
||||
"url": "https://api.familyinfo.buckinghamshire.gov.uk/api/v1/services",
|
||||
"name": "/services",
|
||||
"description": "",
|
||||
"method": "GET",
|
||||
"body": {},
|
||||
"parameters": [],
|
||||
"headers": [
|
||||
{
|
||||
"name": "User-Agent",
|
||||
"value": "insomnia/0.2.2"
|
||||
}
|
||||
],
|
||||
"authentication": {},
|
||||
"metaSortKey": -1734101284292,
|
||||
"isPrivate": false,
|
||||
"settingStoreCookies": true,
|
||||
"settingSendCookies": true,
|
||||
"settingDisableRenderRequestBody": false,
|
||||
"settingEncodeUrl": true,
|
||||
"settingRebuildPath": true,
|
||||
"settingFollowRedirects": "global",
|
||||
"segmentParams": [],
|
||||
"_type": "request"
|
||||
},
|
||||
{
|
||||
"_id": "fld_7db16d60402144d2ba30852b58a5bd82",
|
||||
"parentId": "wrk_b6699c43c9a1492787836ea5019c5982",
|
||||
"modified": 1734101277775,
|
||||
"created": 1734101277775,
|
||||
"name": "Buckinghamshire Council",
|
||||
"description": "",
|
||||
"environment": {},
|
||||
"environmentPropertyOrder": null,
|
||||
"metaSortKey": -1734101277775,
|
||||
"_type": "request_group"
|
||||
},
|
||||
{
|
||||
"_id": "req_c2e99c36c4ec4deb9c283e6a47076f3d",
|
||||
"parentId": "fld_ab7b6940dd0c44049d4e41af5c68637c",
|
||||
"modified": 1734101205414,
|
||||
"created": 1734101149071,
|
||||
"url": "https://bristol.openplace.directory/o/ServiceDirectoryService/v2/hservices",
|
||||
"name": "/services",
|
||||
"description": "",
|
||||
"method": "GET",
|
||||
"body": {},
|
||||
"parameters": [],
|
||||
"headers": [
|
||||
{
|
||||
"name": "User-Agent",
|
||||
"value": "insomnia/0.2.2"
|
||||
}
|
||||
],
|
||||
"authentication": {},
|
||||
"metaSortKey": -1734101205391,
|
||||
"isPrivate": false,
|
||||
"settingStoreCookies": true,
|
||||
"settingSendCookies": true,
|
||||
"settingDisableRenderRequestBody": false,
|
||||
"settingEncodeUrl": true,
|
||||
"settingRebuildPath": true,
|
||||
"settingFollowRedirects": "global",
|
||||
"segmentParams": [],
|
||||
"_type": "request"
|
||||
},
|
||||
{
|
||||
"_id": "fld_ab7b6940dd0c44049d4e41af5c68637c",
|
||||
"parentId": "wrk_b6699c43c9a1492787836ea5019c5982",
|
||||
"modified": 1734101191560,
|
||||
"created": 1734101191560,
|
||||
"name": "Bristol Council",
|
||||
"description": "",
|
||||
"environment": {},
|
||||
"environmentPropertyOrder": null,
|
||||
"metaSortKey": -1734101191560,
|
||||
"_type": "request_group"
|
||||
},
|
||||
{
|
||||
"_id": "env_de6c835521010ba26be4220f11dc5a6f5069ddf3",
|
||||
"parentId": "wrk_b6699c43c9a1492787836ea5019c5982",
|
||||
"modified": 1734101147098,
|
||||
"created": 1734101147098,
|
||||
"name": "Base Environment",
|
||||
"data": {},
|
||||
"dataPropertyOrder": null,
|
||||
"color": null,
|
||||
"isPrivate": false,
|
||||
"metaSortKey": 1734101147098,
|
||||
"_type": "environment"
|
||||
},
|
||||
{
|
||||
"_id": "jar_de6c835521010ba26be4220f11dc5a6f5069ddf3",
|
||||
"parentId": "wrk_b6699c43c9a1492787836ea5019c5982",
|
||||
"modified": 1734101376960,
|
||||
"created": 1734101147098,
|
||||
"name": "Default Jar",
|
||||
"cookies": [
|
||||
{
|
||||
"key": "JSESSIONID",
|
||||
"value": "1B079D52EEAE9DBC6783C96D71EDF399",
|
||||
"domain": "bristol.openplace.directory",
|
||||
"path": "/",
|
||||
"secure": true,
|
||||
"httpOnly": true,
|
||||
"hostOnly": true,
|
||||
"creation": "2024-12-13T14:46:10.292Z",
|
||||
"lastAccessed": "2024-12-13T14:46:10.292Z",
|
||||
"id": "4630461386775335"
|
||||
},
|
||||
{
|
||||
"key": "AWSELB",
|
||||
"value": "2D870D5F0E8860F2EF95DB74F9FDC976417AF8CE68BB383FC3865E321F4838F1CD6B5C9AF23B387FD2E2AAA3FA0C38F5BEF0D9172D5C9046BC720034DB7DD4D5154F9B072EE2AA12398CEDCEC6C4A209C1F79F782A",
|
||||
"domain": "bristol.openplace.directory",
|
||||
"path": "/",
|
||||
"secure": true,
|
||||
"httpOnly": true,
|
||||
"hostOnly": true,
|
||||
"creation": "2024-12-13T14:46:10.294Z",
|
||||
"lastAccessed": "2024-12-13T14:46:10.294Z",
|
||||
"id": "12139281671488988"
|
||||
},
|
||||
{
|
||||
"key": "JSESSIONID",
|
||||
"value": "DF04FB103146B7994E88BCC9E7B72F31",
|
||||
"domain": "northlincs.openplace.directory",
|
||||
"path": "/",
|
||||
"secure": true,
|
||||
"httpOnly": true,
|
||||
"hostOnly": true,
|
||||
"creation": "2024-12-13T14:48:41.552Z",
|
||||
"lastAccessed": "2024-12-13T14:48:41.552Z",
|
||||
"id": "1023352684773664"
|
||||
},
|
||||
{
|
||||
"key": "AWSELB",
|
||||
"value": "2D870D5F0E8860F2EF95DB74F9FDC976417AF8CE6809D3AEDDCA61FB82F1F2060AEEA8F11C6541C0E2BA34496CB5FB94A03D9314795C9046BC720034DB7DD4D5154F9B072EE2AA12398CEDCEC6C4A209C1F79F782A",
|
||||
"domain": "northlincs.openplace.directory",
|
||||
"path": "/",
|
||||
"secure": true,
|
||||
"httpOnly": true,
|
||||
"hostOnly": true,
|
||||
"creation": "2024-12-13T14:48:41.552Z",
|
||||
"lastAccessed": "2024-12-13T14:48:41.552Z",
|
||||
"id": "8515615299671728"
|
||||
},
|
||||
{
|
||||
"key": "JSESSIONID",
|
||||
"value": "0144C070FC1D17CE851A8B6A2C8F5B61",
|
||||
"domain": "penninelancs.openplace.directory",
|
||||
"path": "/",
|
||||
"secure": true,
|
||||
"httpOnly": true,
|
||||
"hostOnly": true,
|
||||
"creation": "2024-12-13T14:49:36.958Z",
|
||||
"lastAccessed": "2024-12-13T14:49:36.958Z",
|
||||
"id": "7743775872235581"
|
||||
},
|
||||
{
|
||||
"key": "AWSELB",
|
||||
"value": "2D870D5F0E8860F2EF95DB74F9FDC976417AF8CE68BFA6CBA6790DEEA2DD2A9AC9FAFF5B7776B09675239D14A769B864D1960639ED5C9046BC720034DB7DD4D5154F9B072EE2AA12398CEDCEC6C4A209C1F79F782A",
|
||||
"domain": "penninelancs.openplace.directory",
|
||||
"path": "/",
|
||||
"secure": true,
|
||||
"httpOnly": true,
|
||||
"hostOnly": true,
|
||||
"creation": "2024-12-13T14:49:36.959Z",
|
||||
"lastAccessed": "2024-12-13T14:49:36.959Z",
|
||||
"id": "8352805125006024"
|
||||
}
|
||||
],
|
||||
"_type": "cookie_jar"
|
||||
}
|
||||
]
|
||||
}
|
||||
25
package-lock.json
generated
25
package-lock.json
generated
|
|
@ -1,22 +1,25 @@
|
|||
{
|
||||
"name": "or-uk-data-quality-checker",
|
||||
"version": "1.0.0",
|
||||
"name": "fh-or-tester",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "or-uk-data-quality-checker",
|
||||
"version": "1.0.0",
|
||||
"name": "fh-or-tester",
|
||||
"dependencies": {
|
||||
"commander": "^12.1.0"
|
||||
"uuid": "^11.0.3"
|
||||
}
|
||||
},
|
||||
"node_modules/commander": {
|
||||
"version": "12.1.0",
|
||||
"resolved": "https://registry.npmjs.org/commander/-/commander-12.1.0.tgz",
|
||||
"integrity": "sha512-Vw8qHK3bZM9y/P10u3Vib8o/DdkvA2OtPtZvD871QKjy74Wj1WSKFILMPRPSdUSx5RFK1arlJzEtA4PkFgnbuA==",
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
"node_modules/uuid": {
|
||||
"version": "11.0.3",
|
||||
"resolved": "https://registry.npmjs.org/uuid/-/uuid-11.0.3.tgz",
|
||||
"integrity": "sha512-d0z310fCWv5dJwnX1Y/MncBAqGMKEzlBb1AOf7z9K8ALnd0utBX/msg/fA0+sbyN1ihbMsLhrBlnl1ak7Wa0rg==",
|
||||
"funding": [
|
||||
"https://github.com/sponsors/broofa",
|
||||
"https://github.com/sponsors/ctavan"
|
||||
],
|
||||
"license": "MIT",
|
||||
"bin": {
|
||||
"uuid": "dist/esm/bin/uuid"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,9 +1,8 @@
|
|||
{
|
||||
"name": "or-uk-data-quality-checker",
|
||||
"description": "A tool to fetch and validate the quality of data from Open Referral UK APIs.",
|
||||
"version": "1.0.0",
|
||||
"main": "index.js",
|
||||
"dependencies": {
|
||||
"commander": "^12.1.0"
|
||||
"scripts": {
|
||||
"fetch": "node script-fetch.js",
|
||||
"eval": "node script-eval.js"
|
||||
}
|
||||
}
|
||||
|
|
|
|||
112
script-eval.js
Normal file
112
script-eval.js
Normal file
|
|
@ -0,0 +1,112 @@
|
|||
const fs = require('fs');
|
||||
|
||||
(async () => {
|
||||
const path = process.argv[0];
|
||||
if (!path) {
|
||||
throw new Error("Path not provided.");
|
||||
}
|
||||
|
||||
// Load JSON from the path provided in the first CLI arg
|
||||
const services = await loadData(process.argv[2]);
|
||||
// Run the validators against every service in the list
|
||||
const results = services.map(service => {
|
||||
return {
|
||||
id: service.id,
|
||||
...Object.keys(validators).reduce((acc, name) => {
|
||||
acc[name] = validators[name].fn(service);
|
||||
return acc;
|
||||
}, {})
|
||||
};
|
||||
});
|
||||
|
||||
console.table(results);
|
||||
|
||||
for (let validatorName in Object.keys(validators)) {
|
||||
const validator = validators[validatorName];
|
||||
const total = results.filter(x => x[validator])
|
||||
}
|
||||
|
||||
|
||||
// const totalWithValidID = results.filter(x => x.hasValidID).length;
|
||||
// console.log(`# of services with a valid ID: ${totalWithValidID}/${results.length} (${Math.round((totalWithValidID/results.length)*100)}%)`);
|
||||
|
||||
// const totalWithValidStatus = results.filter(x => x.hasValidStatus).length;
|
||||
// console.log(`# of services with a valid status: ${totalWithValidStatus}/${results.length} (${Math.round((totalWithValidStatus/results.length)*100)}%)`);
|
||||
|
||||
// const totalWithValidName = results.filter(x => x.hasValidName).length;
|
||||
// console.log(`# of services with a valid name: ${totalWithValidName}/${results.length} (${Math.round((totalWithValidName/results.length)*100)}%)`);
|
||||
|
||||
// const totalWithValidDescription = results.filter(x => x.hasValidDescription).length;
|
||||
// console.log(`# of services with a valid description: ${totalWithValidDescription}/${results.length} (${Math.round((totalWithValidDescription/results.length)*100)}%)`);
|
||||
|
||||
// const totalWithValidURL = results.filter(x => x.hasValidURL).length;
|
||||
// console.log(`# of services with a valid URL: ${totalWithValidURL}/${results.length} (${Math.round((totalWithValidURL/results.length)*100)}%)`);
|
||||
|
||||
// const totalWithValidOrganisation = results.filter(x => x.hasValidOrganisation).length;
|
||||
// console.log(`# of services with a valid organisation: ${totalWithValidOrganisation}/${results.length} (${Math.round((totalWithValidOrganisation/results.length)*100)}%)`);
|
||||
|
||||
// const totalWithValidContact = results.filter(x => x.hasValidContact).length;
|
||||
// console.log(`# of services with a valid contact: ${totalWithValidContact}/${results.length} (${Math.round((totalWithValidContact/results.length)*100)}%)`);
|
||||
|
||||
// console.log("--------------------");
|
||||
|
||||
// const totalUsableServices = results.filter(x =>
|
||||
// x.hasValidID &&
|
||||
// x.hasValidStatus &&
|
||||
// x.hasValidName &&
|
||||
// x.hasValidDescription &&
|
||||
// x.hasValidURL &&
|
||||
// x.hasValidOrganisation &&
|
||||
// x.hasValidContact).length;
|
||||
|
||||
// console.log(`# of valid usable services: ${totalUsableServices}/${results.length} (${Math.round((totalUsableServices/results.length)*100)}%)`);
|
||||
|
||||
})();
|
||||
|
||||
async function loadData(path) {
|
||||
const text = await fs.promises.readFile(path, {
|
||||
encoding: "utf-8"
|
||||
});
|
||||
|
||||
return JSON.parse(text);
|
||||
}
|
||||
|
||||
const validators = {
|
||||
id: {
|
||||
fn: (s) => {
|
||||
return /^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$/i.test(s.id.trim());
|
||||
},
|
||||
desc: "# of services with a valid ID"
|
||||
},
|
||||
status: {
|
||||
fn: (s) => s.status === "active",
|
||||
desc: "# of services with a valid status"
|
||||
},
|
||||
name: {
|
||||
fn: (s) => typeof s.name === "string" && s.name.length > 0,
|
||||
desc: "# of services with a valid name"
|
||||
},
|
||||
description: {
|
||||
fn: (s) => typeof s.description === "string" && s.description.length > 0,
|
||||
desc: "# of services with a valid description"
|
||||
},
|
||||
url: {
|
||||
fn: (s) => {
|
||||
try {
|
||||
new URL(s.url);
|
||||
return true;
|
||||
} catch (_) {
|
||||
return false;
|
||||
}
|
||||
},
|
||||
desc: "# of services with a valid URL"
|
||||
},
|
||||
org: {
|
||||
fn: (s) => !!s.organization?.id && !!s.organization?.name,
|
||||
desc: "# of services with a valid organisation"
|
||||
},
|
||||
contact: {
|
||||
fn: (s) => !!s.email?.length > 0,
|
||||
desc: "# of services with a valid contact"
|
||||
}
|
||||
};
|
||||
68
script-fetch.js
Normal file
68
script-fetch.js
Normal file
|
|
@ -0,0 +1,68 @@
|
|||
const { wait } = require("./helpers");
|
||||
const insomnium = require("./insomnium.json");
|
||||
const fs = require("fs");
|
||||
|
||||
const WAIT_BETWEEN_REQS_MS = 500;
|
||||
const PER_PAGE=500;
|
||||
|
||||
const groups = insomnium.resources
|
||||
.filter(res => res._type === "request_group");
|
||||
|
||||
const las = insomnium.resources
|
||||
.filter(res =>
|
||||
res._type === "request" &&
|
||||
res.name === "/services")
|
||||
.map(res => ({
|
||||
group: groups.find(group => group._id == res.parentId),
|
||||
res
|
||||
}));
|
||||
|
||||
(async() => {
|
||||
console.log('Fetching LAs...');
|
||||
|
||||
for (let la of las) {
|
||||
const laName = la.group.name;
|
||||
|
||||
console.log(`Fetching services for ${laName}...`);
|
||||
const services = await fetchServices(la.res.url);
|
||||
console.log(`Found ${services.length} services for ${laName}!`);
|
||||
|
||||
await writeServices(laName, services);
|
||||
}
|
||||
})();
|
||||
|
||||
async function fetchServices(ep) {
|
||||
let page = 1;
|
||||
let services = [];
|
||||
|
||||
do {
|
||||
const pagedEp = `${ep}?page=${page}&per_page=${PER_PAGE}`;
|
||||
const resp = await fetch(pagedEp);
|
||||
console.log(pagedEp);
|
||||
|
||||
if (!resp.ok) {
|
||||
console.warn(`Could not fetch services for ${ep}. Skipping. (Resp code ${resp.status})`);
|
||||
return [];
|
||||
}
|
||||
|
||||
const json = await resp.json();
|
||||
|
||||
services = [...services, ...json.content];
|
||||
|
||||
if (json.last == true) {
|
||||
console.log("All services found!")
|
||||
break;
|
||||
}
|
||||
|
||||
console.log(`Loaded ${services.length} of total ${json.totalElements}...`);
|
||||
|
||||
page++;
|
||||
await wait(WAIT_BETWEEN_REQS_MS);
|
||||
} while (true);
|
||||
|
||||
return services;
|
||||
}
|
||||
|
||||
async function writeServices(name, services) {
|
||||
await fs.promises.writeFile(`data/${name}.json`, JSON.stringify(services, null, 2));
|
||||
}
|
||||
|
|
@ -1,36 +0,0 @@
|
|||
// const hasValidId = (s) =>
|
||||
// /^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$/i.test(s.id.trim());
|
||||
|
||||
const hasValidId = (s) => !!s.id?.trim();
|
||||
|
||||
const hasValidName = (s) => typeof s.name === "string" && s.name.length > 0;
|
||||
|
||||
const hasValidDescription = (s) => typeof s.description === "string" && s.description.length > 0;
|
||||
|
||||
const hasValidUrl = (s) => {
|
||||
try {
|
||||
new URL(s.url);
|
||||
return true;
|
||||
} catch (_) {
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
const hasValidOrganisation = (s) => !!s.organization?.id && !!s.organization?.name;
|
||||
|
||||
const hasValidContact = (s) =>
|
||||
s.email?.length > 0 ||
|
||||
(s.contacts?.[0]?.phones?.some(x => !!x.number) ?? false);
|
||||
|
||||
|
||||
const hasValidStatus = (s) => s.status === "active";
|
||||
|
||||
module.exports = {
|
||||
hasValidId,
|
||||
hasValidName,
|
||||
hasValidDescription,
|
||||
hasValidUrl,
|
||||
hasValidOrganisation,
|
||||
hasValidContact,
|
||||
hasValidStatus
|
||||
};
|
||||
Loading…
Add table
Reference in a new issue