253 lines
8 KiB
TypeScript
253 lines
8 KiB
TypeScript
import {
|
|
DOMParser,
|
|
} from "https://deno.land/x/deno_dom@v0.1.45/deno-dom-wasm.ts";
|
|
import { writeCSV } from "https://deno.land/x/csv@v0.9.2/mod.ts";
|
|
|
|
import {
|
|
NSDAMemberLookup,
|
|
NSDAMemberPointEntry,
|
|
NSDAMemberSearchResult,
|
|
TabroomEntry,
|
|
} from "./types.ts";
|
|
|
|
import { overrides } from "./constants.ts";
|
|
|
|
// Load auth
|
|
const nsdaAuth = await Deno.readFile("./auth.txt").then((a) =>
|
|
new TextDecoder().decode(a)
|
|
);
|
|
|
|
// Fetch and parse tabroom USX entries
|
|
const tabroomHtml = await fetch(
|
|
"https://www.tabroom.com/index/tourn/fields.mhtml?tourn_id=30061&event_id=279207",
|
|
).then((r) => r.text());
|
|
|
|
const dom = new DOMParser().parseFromString(
|
|
tabroomHtml,
|
|
"text/html",
|
|
)!;
|
|
|
|
const entries: TabroomEntry[] = [];
|
|
|
|
for (const row of dom.querySelectorAll("tr")) {
|
|
const [institution, location, entry, code] = [...row.childNodes]
|
|
.filter((n) => n.nodeName === "TD")
|
|
.map((n) => n.textContent.trim());
|
|
// Skip invalid entries
|
|
if (!institution || !location || !entry || !code || entry === "Names TBA") {
|
|
continue;
|
|
}
|
|
|
|
entries.push({
|
|
institution,
|
|
location,
|
|
entry,
|
|
code,
|
|
});
|
|
}
|
|
|
|
// Lookup on NSDA
|
|
const nsdaSearchResults = await Promise
|
|
.all(
|
|
entries.map(async (e) => {
|
|
const override = overrides[e.entry];
|
|
|
|
let nsdaLookup: NSDAMemberSearchResult[] = await fetch(
|
|
`https://api.speechanddebate.org/v2/search?${
|
|
new URLSearchParams({
|
|
q: override?.name ?? e.entry,
|
|
type: "members",
|
|
}).toString()
|
|
}`,
|
|
{
|
|
headers: {
|
|
authorization: nsdaAuth,
|
|
},
|
|
},
|
|
).then((r) => r.json());
|
|
|
|
// Short circuit on overrides
|
|
if (override !== undefined) {
|
|
const result = nsdaLookup.find((e) => e.id === override.id);
|
|
if (result !== undefined) {
|
|
return { tabroom: e, nsdaSearchResult: result };
|
|
} else {
|
|
console.error(`ID override failed for ${e.entry} (${override.id})!`);
|
|
return null;
|
|
}
|
|
}
|
|
|
|
// Do our best to filter the results down to one
|
|
// Start by filtering anything but HS competitors and status === 1
|
|
// (status 2 appeared on a duplicate, so I assume that marks inactive entries?)
|
|
nsdaLookup = nsdaLookup.filter((ee) =>
|
|
ee.realm === "hs" && ee.status === 1
|
|
);
|
|
// Then, if that doesn't work, filter by state
|
|
if (nsdaLookup.length > 1) {
|
|
nsdaLookup = nsdaLookup.filter((ee) =>
|
|
ee.school_state?.toLowerCase() ===
|
|
e.location.replace("/US", "").toLowerCase()
|
|
);
|
|
}
|
|
// At this point, collapse equivelent results just in case we *have* found one result
|
|
if (nsdaLookup.length > 1) {
|
|
const collapsed: NSDAMemberSearchResult[] = [];
|
|
for (const entry of nsdaLookup) {
|
|
if (!collapsed.some((e) => e.id === entry.id)) {
|
|
collapsed.push(entry);
|
|
}
|
|
}
|
|
nsdaLookup = collapsed;
|
|
}
|
|
// Finally, if none of the past filters worked, attempt a school filter
|
|
let schoolFilterUsed = false;
|
|
if (nsdaLookup.length > 1) {
|
|
schoolFilterUsed = true;
|
|
|
|
nsdaLookup = nsdaLookup.filter((ee) =>
|
|
ee.school_name.replace(/[^a-zA-Z0-9 -]/g, "").toLowerCase().includes(
|
|
e.institution.replace(/[^a-zA-Z0-9 -]/g, "").toLowerCase(),
|
|
) ||
|
|
e.institution.replace(/[^a-zA-Z0-9 -]/g, "").toLowerCase().includes(
|
|
ee.school_name.replace(/[^a-zA-Z0-9 -]/g, "").toLowerCase(),
|
|
)
|
|
);
|
|
}
|
|
|
|
if (nsdaLookup.length < 1) {
|
|
console.error(
|
|
`No results found for entry "${e.entry}" (with code "${e.code}"), from ${e.institution} in ${
|
|
e.location.replace("/US", "")
|
|
}.`,
|
|
);
|
|
if (schoolFilterUsed) console.error("SCHOOL-BASED FILTER WAS USED!");
|
|
return null;
|
|
} else if (nsdaLookup.length > 1) {
|
|
console.error(
|
|
`Multiple entries found for entry "${e.entry}" (with code "${e.code}"), from ${e.institution} in ${
|
|
e.location.replace("/US", "")
|
|
}!`,
|
|
);
|
|
console.error("---------------------------------------------");
|
|
console.error(nsdaLookup);
|
|
console.error("---------------------------------------------");
|
|
return null;
|
|
} else {
|
|
return { tabroom: e, nsdaSearchResult: nsdaLookup[0] };
|
|
}
|
|
}),
|
|
).then((p) =>
|
|
p.filter((e) => e !== null) as {
|
|
tabroom: TabroomEntry;
|
|
nsdaSearchResult: NSDAMemberSearchResult;
|
|
}[]
|
|
);
|
|
|
|
// Convert NSDA searches to true NSDA member objects and point entries to get actual information
|
|
const nsdaLookups = await Promise.all(
|
|
nsdaSearchResults.map(async ({ tabroom, nsdaSearchResult }) => ({
|
|
tabroom,
|
|
nsdaSearchResult,
|
|
nsdaMemberLookup: await fetch(
|
|
"https://api.speechanddebate.org/v2/members/" + nsdaSearchResult.id,
|
|
{ headers: { authorization: nsdaAuth } },
|
|
).then((r) => r.json()) as NSDAMemberLookup,
|
|
nsdaPointsLookup: await fetch(
|
|
"https://api.speechanddebate.org/v2/members/" + nsdaSearchResult.id +
|
|
"/points",
|
|
{ headers: { authorization: nsdaAuth } },
|
|
).then((r) => r.json()) as NSDAMemberPointEntry[],
|
|
})),
|
|
);
|
|
|
|
// Analyze and display aggregate fetched information
|
|
console.log(
|
|
`Represented states:
|
|
${
|
|
Object.entries(nsdaLookups.reduce(
|
|
(acc, l) => ({
|
|
...acc,
|
|
[l.tabroom.location.replace("/US", "")]:
|
|
(acc[l.tabroom.location.replace("/US", "")] ?? 0) + 1,
|
|
}),
|
|
{} as Record<string, number>,
|
|
)).sort((a, b) => b[1] - a[1]).map(([state, n]) => "- " + state + ": " + n)
|
|
.join("\n")
|
|
}
|
|
Total entries: ${nsdaLookups.length}
|
|
Total schools: ${
|
|
[...new Set(nsdaLookups.map((l) => l.nsdaSearchResult.school_name))].length
|
|
}`,
|
|
);
|
|
|
|
// Collect information into a spreadsheet format
|
|
const sheet = [
|
|
[
|
|
"Last points entry",
|
|
"First name",
|
|
"Middle name",
|
|
"Last name",
|
|
"State",
|
|
"School",
|
|
"Extemp Points",
|
|
"Year Extemp Points",
|
|
"Extemp tournaments",
|
|
"Year Extemp Tournaments",
|
|
"Average Placing",
|
|
"Year Average Placing",
|
|
],
|
|
...nsdaLookups.map((l) => [
|
|
l.nsdaMemberLookup.last_points_entry,
|
|
l.nsdaMemberLookup.first,
|
|
l.nsdaMemberLookup.middle ?? "",
|
|
l.nsdaMemberLookup.last,
|
|
l.nsdaSearchResult.school_state ?? l.tabroom.location.replace("/US", ""),
|
|
l.nsdaSearchResult.school_name,
|
|
l.nsdaPointsLookup.filter((e) =>
|
|
e.category_id === 202 /* Extemporaneous speech event ID */
|
|
).reduce((acc, e) => acc + e.points, 0).toString(),
|
|
l.nsdaPointsLookup.filter((e) =>
|
|
e.category_id === 202 && /* Extemporaneous speech event ID */
|
|
new Date(e.start).getTime() > new Date(2023, 6, 0).getTime()
|
|
).reduce((acc, e) => acc + e.points, 0).toString(),
|
|
l.nsdaPointsLookup.filter((e) =>
|
|
e.category_id === 202 /* Extemporaneous speech event ID */
|
|
).reduce(
|
|
(acc, e) => [...acc, ...(acc.includes(e.tourn_id) ? [] : [e.tourn_id])],
|
|
[] as number[],
|
|
).length.toString(),
|
|
l.nsdaPointsLookup.filter((e) =>
|
|
e.category_id === 202 && /* Extemporaneous speech event ID */
|
|
new Date(e.start).getTime() > new Date(2023, 6, 0).getTime()
|
|
).reduce(
|
|
(acc, e) => [...acc, ...(acc.includes(e.tourn_id) ? [] : [e.tourn_id])],
|
|
[] as number[],
|
|
).length.toString(),
|
|
(Math.round(
|
|
l.nsdaPointsLookup.filter((e) =>
|
|
e.category_id === 202 /* Extemporaneous speech event ID */
|
|
).flatMap((e) => (JSON.parse(e.result) as { ranks: number[] }).ranks)
|
|
.reduce((acc, v, _, a) => (acc + v / a.length), 0) * 1000,
|
|
) / 1000).toString(),
|
|
(Math.round(
|
|
l.nsdaPointsLookup.filter((e) =>
|
|
e.category_id === 202 && /* Extemporaneous speech event ID */
|
|
new Date(e.start).getTime() > new Date(2023, 6, 0).getTime()
|
|
).flatMap((e) => (JSON.parse(e.result) as { ranks: number[] }).ranks)
|
|
.reduce((acc, v, _, a) => (acc + v / a.length), 0) * 1000,
|
|
) / 1000).toString(),
|
|
]),
|
|
];
|
|
|
|
await writeCSV(
|
|
await Deno.open("./out.csv", {
|
|
write: true,
|
|
create: true,
|
|
truncate: true,
|
|
}),
|
|
sheet,
|
|
{
|
|
columnSeparator: "\t",
|
|
},
|
|
);
|