Initial commit
This commit is contained in:
commit
8f92e078bf
GPG key ID:
2813440C772555A4
4 changed files with 348 additions and 0 deletions
2
.gitignore
vendored
Normal file
2
.gitignore
vendored
Normal file
|
|
@ -0,0 +1,2 @@
|
|||
auth.txt
|
||||
out.csv
|
||||
3
.vscode/settings.json
vendored
Normal file
3
.vscode/settings.json
vendored
Normal file
|
|
@ -0,0 +1,3 @@
|
|||
{
|
||||
"deno.enable": true
|
||||
}
|
||||
221
cli.ts
Normal file
221
cli.ts
Normal file
|
|
@ -0,0 +1,221 @@
|
|||
import {
|
||||
DOMParser,
|
||||
} from "https://deno.land/x/deno_dom@v0.1.45/deno-dom-wasm.ts";
|
||||
import { writeCSV } from "https://deno.land/x/csv@v0.9.2/mod.ts";
|
||||
|
||||
import {
|
||||
NSDAMemberLookup,
|
||||
NSDAMemberPointEntry,
|
||||
NSDAMemberSearchResult,
|
||||
TabroomEntry,
|
||||
} from "./types.ts";
|
||||
|
||||
// Load auth
|
||||
const nsdaAuth = await Deno.readFile("./auth.txt").then((a) =>
|
||||
new TextDecoder().decode(a)
|
||||
);
|
||||
|
||||
// Fetch and parse tabroom USX entries
|
||||
const tabroomHtml = await fetch(
|
||||
"https://www.tabroom.com/index/tourn/fields.mhtml?tourn_id=30061&event_id=279207",
|
||||
).then((r) => r.text());
|
||||
|
||||
const dom = new DOMParser().parseFromString(
|
||||
tabroomHtml,
|
||||
"text/html",
|
||||
)!;
|
||||
|
||||
const entries: TabroomEntry[] = [];
|
||||
|
||||
for (const row of dom.querySelectorAll("tr")) {
|
||||
const [institution, location, entry, code] = [...row.childNodes]
|
||||
.filter((n) => n.nodeName === "TD")
|
||||
.map((n) => n.textContent.trim());
|
||||
// Skip invalid entries
|
||||
if (!institution || !location || !entry || !code) continue;
|
||||
|
||||
entries.push({
|
||||
institution,
|
||||
location,
|
||||
entry,
|
||||
code,
|
||||
});
|
||||
}
|
||||
|
||||
// Lookup on NSDA
|
||||
const nsdaSearchResults = await Promise
|
||||
.all(
|
||||
entries.map(async (e) => {
|
||||
let nsdaLookup: NSDAMemberSearchResult[] = await fetch(
|
||||
`https://api.speechanddebate.org/v2/search?${
|
||||
new URLSearchParams({
|
||||
q: e.entry,
|
||||
type: "members",
|
||||
}).toString()
|
||||
}`,
|
||||
{
|
||||
headers: {
|
||||
authorization: nsdaAuth,
|
||||
},
|
||||
},
|
||||
).then((r) => r.json());
|
||||
|
||||
// Do our best to filter the results down to one
|
||||
// Start by filtering anything but HS competitors and status === 1
|
||||
// (status 2 appeared on a duplicate, so I assume that marks inactive entries?)
|
||||
nsdaLookup = nsdaLookup.filter((ee) =>
|
||||
ee.realm === "hs" && ee.status === 1
|
||||
);
|
||||
// Then, if that doesn't work, filter by state
|
||||
if (nsdaLookup.length > 1) {
|
||||
nsdaLookup = nsdaLookup.filter((ee) =>
|
||||
ee.school_state.toLowerCase() ===
|
||||
e.location.replace("/US", "").toLowerCase()
|
||||
);
|
||||
}
|
||||
// Finally, if none of the past filters worked, attempt a school filter
|
||||
let schoolFilterUsed = false;
|
||||
if (nsdaLookup.length > 1) {
|
||||
schoolFilterUsed = true;
|
||||
|
||||
nsdaLookup = nsdaLookup.filter((ee) =>
|
||||
ee.school_name.toLowerCase().includes(e.institution.toLowerCase())
|
||||
);
|
||||
}
|
||||
|
||||
if (nsdaLookup.length < 1) {
|
||||
console.error(
|
||||
`No results found for entry "${e.entry}" (with code "${e.code}"), from ${e.institution} in ${
|
||||
e.location.replace("/US", "")
|
||||
}.`,
|
||||
);
|
||||
if (schoolFilterUsed) console.error("SCHOOL-BASED FILTER WAS USED!");
|
||||
return null;
|
||||
} else if (nsdaLookup.length > 1) {
|
||||
console.error(
|
||||
`Multiple entries found for entry "${e.entry}" (with code "${e.code}"), from ${e.institution} in ${
|
||||
e.location.replace("/US", "")
|
||||
}!`,
|
||||
);
|
||||
console.error("---------------------------------------------");
|
||||
console.error(nsdaLookup);
|
||||
console.error("---------------------------------------------");
|
||||
return null;
|
||||
} else {
|
||||
return { tabroom: e, nsdaSearchResult: nsdaLookup[0] };
|
||||
}
|
||||
}),
|
||||
).then((p) =>
|
||||
p.filter((e) => e !== null) as {
|
||||
tabroom: TabroomEntry;
|
||||
nsdaSearchResult: NSDAMemberSearchResult;
|
||||
}[]
|
||||
);
|
||||
|
||||
// Convert NSDA searches to true NSDA member objects and point entries to get actual information
|
||||
const nsdaLookups = await Promise.all(
|
||||
nsdaSearchResults.map(async ({ tabroom, nsdaSearchResult }) => ({
|
||||
tabroom,
|
||||
nsdaSearchResult,
|
||||
nsdaMemberLookup: await fetch(
|
||||
"https://api.speechanddebate.org/v2/members/" + nsdaSearchResult.id,
|
||||
{ headers: { authorization: nsdaAuth } },
|
||||
).then((r) => r.json()) as NSDAMemberLookup,
|
||||
nsdaPointsLookup: await fetch(
|
||||
"https://api.speechanddebate.org/v2/members/" + nsdaSearchResult.id +
|
||||
"/points",
|
||||
{ headers: { authorization: nsdaAuth } },
|
||||
).then((r) => r.json()) as NSDAMemberPointEntry[],
|
||||
})),
|
||||
);
|
||||
|
||||
// Analyze and display aggregate fetched information
|
||||
console.log(
|
||||
`Represented states:
|
||||
${
|
||||
Object.entries(nsdaLookups.reduce(
|
||||
(acc, l) => ({
|
||||
...acc,
|
||||
[l.nsdaSearchResult.school_state]:
|
||||
(acc[l.nsdaSearchResult.school_state] ?? 0) + 1,
|
||||
}),
|
||||
{} as Record<string, number>,
|
||||
)).sort((a, b) => b[1] - a[1]).map(([state, n]) => "- " + state + ": " + n)
|
||||
.join("\n")
|
||||
}
|
||||
Total entries: ${nsdaLookups.length}
|
||||
Total schools: ${
|
||||
[...new Set(nsdaLookups.map((l) => l.nsdaSearchResult.school_name))].length
|
||||
}`,
|
||||
);
|
||||
|
||||
// Collect information into a spreadsheet format
|
||||
const sheet = [
|
||||
[
|
||||
"Last points entry",
|
||||
"First name",
|
||||
"Middle name",
|
||||
"Last name",
|
||||
"State",
|
||||
"School",
|
||||
"Extemp Points",
|
||||
"Year Extemp Points",
|
||||
"Extemp tournaments",
|
||||
"Year Extemp Tournaments",
|
||||
"Average Placing",
|
||||
"Year Average Placing",
|
||||
],
|
||||
...nsdaLookups.map((l) => [
|
||||
l.nsdaMemberLookup.last_points_entry,
|
||||
l.nsdaMemberLookup.first,
|
||||
l.nsdaMemberLookup.middle ?? "",
|
||||
l.nsdaMemberLookup.last,
|
||||
l.nsdaSearchResult.school_state,
|
||||
l.nsdaSearchResult.school_name,
|
||||
l.nsdaPointsLookup.filter((e) =>
|
||||
e.category_id === 202 /* Extemporaneous speech event ID */
|
||||
).reduce((acc, e) => acc + e.points, 0).toString(),
|
||||
l.nsdaPointsLookup.filter((e) =>
|
||||
e.category_id === 202 && /* Extemporaneous speech event ID */
|
||||
new Date(e.start).getTime() > new Date(2023, 6, 0).getTime()
|
||||
).reduce((acc, e) => acc + e.points, 0).toString(),
|
||||
l.nsdaPointsLookup.filter((e) =>
|
||||
e.category_id === 202 /* Extemporaneous speech event ID */
|
||||
).reduce(
|
||||
(acc, e) => [...acc, ...(acc.includes(e.tourn_id) ? [] : [e.tourn_id])],
|
||||
[] as number[],
|
||||
).length.toString(),
|
||||
l.nsdaPointsLookup.filter((e) =>
|
||||
e.category_id === 202 && /* Extemporaneous speech event ID */
|
||||
new Date(e.start).getTime() > new Date(2023, 6, 0).getTime()
|
||||
).reduce(
|
||||
(acc, e) => [...acc, ...(acc.includes(e.tourn_id) ? [] : [e.tourn_id])],
|
||||
[] as number[],
|
||||
).length.toString(),
|
||||
(Math.round(
|
||||
l.nsdaPointsLookup.filter((e) =>
|
||||
e.category_id === 202 /* Extemporaneous speech event ID */
|
||||
).flatMap((e) => (JSON.parse(e.result) as { ranks: number[] }).ranks)
|
||||
.reduce((acc, v, _, a) => (acc + v / a.length), 0) * 1000,
|
||||
) / 1000).toString(),
|
||||
(Math.round(
|
||||
l.nsdaPointsLookup.filter((e) =>
|
||||
e.category_id === 202 && /* Extemporaneous speech event ID */
|
||||
new Date(e.start).getTime() > new Date(2023, 6, 0).getTime()
|
||||
).flatMap((e) => (JSON.parse(e.result) as { ranks: number[] }).ranks)
|
||||
.reduce((acc, v, _, a) => (acc + v / a.length), 0) * 1000,
|
||||
) / 1000).toString(),
|
||||
]),
|
||||
];
|
||||
|
||||
await writeCSV(
|
||||
await Deno.open("./out.csv", {
|
||||
write: true,
|
||||
create: true,
|
||||
truncate: true,
|
||||
}),
|
||||
sheet,
|
||||
{
|
||||
columnSeparator: "\t",
|
||||
},
|
||||
);
|
||||
122
types.ts
Normal file
122
types.ts
Normal file
|
|
@ -0,0 +1,122 @@
|
|||
export interface NSDAMemberSearchResult {
|
||||
id: number;
|
||||
first: string;
|
||||
last: string;
|
||||
role: string;
|
||||
school_id: number;
|
||||
status: number;
|
||||
realm: string;
|
||||
school_name: string;
|
||||
school_state: string;
|
||||
type: string;
|
||||
name: string;
|
||||
description: string;
|
||||
}
|
||||
|
||||
export interface TabroomEntry {
|
||||
institution: string;
|
||||
location: string;
|
||||
entry: string;
|
||||
code: string;
|
||||
}
|
||||
|
||||
export interface NSDAMemberLookup {
|
||||
person_id: number;
|
||||
first: string;
|
||||
middle: string | null;
|
||||
last: string;
|
||||
disabled: number;
|
||||
created_at: string;
|
||||
points: number;
|
||||
points_service: number;
|
||||
points_service_this_year: number;
|
||||
points_this_year: number;
|
||||
points_last_year: number;
|
||||
last_points_entry: string;
|
||||
citation_points: number;
|
||||
degree_id: number;
|
||||
degree_name: string;
|
||||
to_next_degree: number;
|
||||
diamonds: number | null;
|
||||
"3_diamond": boolean;
|
||||
hof: boolean;
|
||||
aaa: boolean;
|
||||
active: boolean;
|
||||
paid: boolean;
|
||||
paid_latest: string;
|
||||
school_paid: boolean;
|
||||
active_student: boolean;
|
||||
degrees: NSDAMemberDegree[];
|
||||
honors: NSDAMemberHonor[];
|
||||
citations: unknown[];
|
||||
districts_eligible: boolean;
|
||||
}
|
||||
|
||||
export interface NSDAMemberDegree {
|
||||
member_honor_id: number;
|
||||
person_id: number;
|
||||
honor_id: number;
|
||||
description: string;
|
||||
type: string;
|
||||
school_id: number;
|
||||
school_name: string;
|
||||
district_id: number | null;
|
||||
district_name: string | null;
|
||||
state: string | null;
|
||||
points: number;
|
||||
start: string;
|
||||
end: unknown;
|
||||
status: string;
|
||||
recognized: unknown;
|
||||
note: string | null;
|
||||
created_at: string;
|
||||
created_by_id: unknown;
|
||||
}
|
||||
|
||||
export interface NSDAMemberHonor {
|
||||
member_honor_id: number;
|
||||
person_id: number;
|
||||
honor_id: number;
|
||||
description: string;
|
||||
type: string;
|
||||
school_id: number;
|
||||
school_name: string;
|
||||
district_id: number | null;
|
||||
district_name: string | null;
|
||||
state: string | null;
|
||||
points: number | null;
|
||||
start: string;
|
||||
end: unknown;
|
||||
status: string;
|
||||
recognized: unknown;
|
||||
note: string | null;
|
||||
created_at: string;
|
||||
created_by_id: unknown;
|
||||
}
|
||||
|
||||
export interface NSDAMemberPointEntry {
|
||||
id: number;
|
||||
student_id: number;
|
||||
student_name: string;
|
||||
coach_id: number;
|
||||
coach_name: string;
|
||||
school_id: number;
|
||||
category_id: number;
|
||||
points: number;
|
||||
result: string;
|
||||
description: string | null;
|
||||
autopost: boolean;
|
||||
districts: boolean;
|
||||
nationals: boolean;
|
||||
tourn_id: number;
|
||||
tourn_name: string;
|
||||
location: string;
|
||||
state: string;
|
||||
start: string;
|
||||
end: string;
|
||||
source: string;
|
||||
status: number;
|
||||
created_at: string;
|
||||
created_by_id: number | null;
|
||||
created_by: string;
|
||||
}
|
||||
Loading…
Reference in a new issue