LastChanceStalker/cli.ts

import {
  DOMParser,
} from "https://deno.land/x/deno_dom@v0.1.45/deno-dom-wasm.ts";
import { writeCSV } from "https://deno.land/x/csv@v0.9.2/mod.ts";

import {
  NSDAMemberLookup,
  NSDAMemberPointEntry,
  NSDAMemberSearchResult,
  TabroomEntry,
} from "./types.ts";

import { overrides } from "./constants.ts";

// Load auth
const nsdaAuth = await Deno.readFile("./auth.txt").then((a) =>
  new TextDecoder().decode(a)
);

// Fetch and parse tabroom USX entries
const tabroomHtml = await fetch(
  "https://www.tabroom.com/index/tourn/fields.mhtml?tourn_id=30061&event_id=279207",
).then((r) => r.text());

const dom = new DOMParser().parseFromString(
  tabroomHtml,
  "text/html",
)!;

const entries: TabroomEntry[] = [];

for (const row of dom.querySelectorAll("tr")) {
  const [institution, location, entry, code] = [...row.childNodes]
    .filter((n) => n.nodeName === "TD")
    .map((n) => n.textContent.trim());
  // Skip invalid entries
  if (!institution || !location || !entry || !code || entry === "Names TBA") {
    continue;
  }

  entries.push({
    institution,
    location,
    entry,
    code,
  });
}

// Lookup on NSDA
const nsdaSearchResults = await Promise
  .all(
    entries.map(async (e) => {
      const override = overrides[e.entry];

      let nsdaLookup: NSDAMemberSearchResult[] = await fetch(
        `https://api.speechanddebate.org/v2/search?${
          new URLSearchParams({
            q: override?.name ?? e.entry,
            type: "members",
          }).toString()
        }`,
        {
          headers: {
            authorization: nsdaAuth,
          },
        },
      ).then((r) => r.json());

      // Short circuit on overrides
      if (override !== undefined) {
        const result = nsdaLookup.find((e) => e.id === override.id);
        if (result !== undefined) {
          return { tabroom: e, nsdaSearchResult: result };
        } else {
          console.error(`ID override failed for ${e.entry} (${override.id})!`);
          return null;
        }
      }

      // Do our best to filter the results down to one
      // Start by filtering anything but HS competitors and status === 1
      // (status 2 appeared on a duplicate, so I assume that marks inactive entries?)
      nsdaLookup = nsdaLookup.filter((ee) =>
        ee.realm === "hs" && ee.status === 1
      );
      // Then, if that doesn't work, filter by state
      if (nsdaLookup.length > 1) {
        nsdaLookup = nsdaLookup.filter((ee) =>
          ee.school_state?.toLowerCase() ===
            e.location.replace("/US", "").toLowerCase()
        );
      }
      // At this point, collapse equivelent results just in case we *have* found one result
      if (nsdaLookup.length > 1) {
        const collapsed: NSDAMemberSearchResult[] = [];
        for (const entry of nsdaLookup) {
          if (!collapsed.some((e) => e.id === entry.id)) {
            collapsed.push(entry);
          }
        }
        nsdaLookup = collapsed;
      }
      // Finally, if none of the past filters worked, attempt a school filter
      let schoolFilterUsed = false;
      if (nsdaLookup.length > 1) {
        schoolFilterUsed = true;

        nsdaLookup = nsdaLookup.filter((ee) =>
          ee.school_name.replace(/[^a-zA-Z0-9 -]/g, "").toLowerCase().includes(
            e.institution.replace(/[^a-zA-Z0-9 -]/g, "").toLowerCase(),
          ) ||
          e.institution.replace(/[^a-zA-Z0-9 -]/g, "").toLowerCase().includes(
            ee.school_name.replace(/[^a-zA-Z0-9 -]/g, "").toLowerCase(),
          )
        );
      }

      if (nsdaLookup.length < 1) {
        console.error(
          `No results found for entry "${e.entry}" (with code "${e.code}"), from ${e.institution} in ${
            e.location.replace("/US", "")
          }.`,
        );
        if (schoolFilterUsed) console.error("SCHOOL-BASED FILTER WAS USED!");
        return null;
      } else if (nsdaLookup.length > 1) {
        console.error(
          `Multiple entries found for entry "${e.entry}" (with code "${e.code}"), from ${e.institution} in ${
            e.location.replace("/US", "")
          }!`,
        );
        console.error("---------------------------------------------");
        console.error(nsdaLookup);
        console.error("---------------------------------------------");
        return null;
      } else {
        return { tabroom: e, nsdaSearchResult: nsdaLookup[0] };
      }
    }),
  ).then((p) =>
    p.filter((e) => e !== null) as {
      tabroom: TabroomEntry;
      nsdaSearchResult: NSDAMemberSearchResult;
    }[]
  );

// Convert NSDA searches to true NSDA member objects and point entries to get actual information
const nsdaLookups = await Promise.all(
  nsdaSearchResults.map(async ({ tabroom, nsdaSearchResult }) => ({
    tabroom,
    nsdaSearchResult,
    nsdaMemberLookup: await fetch(
      "https://api.speechanddebate.org/v2/members/" + nsdaSearchResult.id,
      { headers: { authorization: nsdaAuth } },
    ).then((r) => r.json()) as NSDAMemberLookup,
    nsdaPointsLookup: await fetch(
      "https://api.speechanddebate.org/v2/members/" + nsdaSearchResult.id +
        "/points",
      { headers: { authorization: nsdaAuth } },
    ).then((r) => r.json()) as NSDAMemberPointEntry[],
  })),
);

// Analyze and display aggregate fetched information
console.log(
  `Represented states:
${
    Object.entries(nsdaLookups.reduce(
      (acc, l) => ({
        ...acc,
        [l.tabroom.location.replace("/US", "")]:
          (acc[l.tabroom.location.replace("/US", "")] ?? 0) + 1,
      }),
      {} as Record<string, number>,
    )).sort((a, b) => b[1] - a[1]).map(([state, n]) => "- " + state + ": " + n)
      .join("\n")
  }
Total entries: ${nsdaLookups.length}
Total schools: ${
    [...new Set(nsdaLookups.map((l) => l.nsdaSearchResult.school_name))].length
  }`,
);

// Collect information into a spreadsheet format
const sheet = [
  [
    "Last points entry",
    "First name",
    "Middle name",
    "Last name",
    "State",
    "School",
    "Extemp Points",
    "Year Extemp Points",
    "Extemp tournaments",
    "Year Extemp Tournaments",
    "Average Placing",
    "Year Average Placing",
  ],
  ...nsdaLookups.map((l) => [
    l.nsdaMemberLookup.last_points_entry,
    l.nsdaMemberLookup.first,
    l.nsdaMemberLookup.middle ?? "",
    l.nsdaMemberLookup.last,
    l.nsdaSearchResult.school_state ?? l.tabroom.location.replace("/US", ""),
    l.nsdaSearchResult.school_name,
    l.nsdaPointsLookup.filter((e) =>
      e.category_id === 202 /* Extemporaneous speech event ID */
    ).reduce((acc, e) => acc + e.points, 0).toString(),
    l.nsdaPointsLookup.filter((e) =>
      e.category_id === 202 && /* Extemporaneous speech event ID */
      new Date(e.start).getTime() > new Date(2023, 6, 0).getTime()
    ).reduce((acc, e) => acc + e.points, 0).toString(),
    l.nsdaPointsLookup.filter((e) =>
      e.category_id === 202 /* Extemporaneous speech event ID */
    ).reduce(
      (acc, e) => [...acc, ...(acc.includes(e.tourn_id) ? [] : [e.tourn_id])],
      [] as number[],
    ).length.toString(),
    l.nsdaPointsLookup.filter((e) =>
      e.category_id === 202 && /* Extemporaneous speech event ID */
      new Date(e.start).getTime() > new Date(2023, 6, 0).getTime()
    ).reduce(
      (acc, e) => [...acc, ...(acc.includes(e.tourn_id) ? [] : [e.tourn_id])],
      [] as number[],
    ).length.toString(),
    (Math.round(
      l.nsdaPointsLookup.filter((e) =>
        e.category_id === 202 /* Extemporaneous speech event ID */
      ).flatMap((e) => (JSON.parse(e.result) as { ranks: number[] }).ranks)
        .reduce((acc, v, _, a) => (acc + v / a.length), 0) * 1000,
    ) / 1000).toString(),
    (Math.round(
      l.nsdaPointsLookup.filter((e) =>
        e.category_id === 202 && /* Extemporaneous speech event ID */
        new Date(e.start).getTime() > new Date(2023, 6, 0).getTime()
      ).flatMap((e) => (JSON.parse(e.result) as { ranks: number[] }).ranks)
        .reduce((acc, v, _, a) => (acc + v / a.length), 0) * 1000,
    ) / 1000).toString(),
  ]),
];

await writeCSV(
  await Deno.open("./out.csv", {
    write: true,
    create: true,
    truncate: true,
  }),
  sheet,
  {
    columnSeparator: "\t",
  },
);