From 387be744800edb933d360866024a3b20a337a363 Mon Sep 17 00:00:00 2001 From: Conan Scott Date: Mon, 23 Mar 2026 23:55:28 +0000 Subject: [PATCH] Fix ingest.py - correct file content --- ingest.py | 257 +++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 256 insertions(+), 1 deletion(-) diff --git a/ingest.py b/ingest.py index be2a3f3..4fe4d44 100644 --- a/ingest.py +++ b/ingest.py @@ -1 +1,256 @@ -IyEvdXNyL2Jpbi9lbnYgcHl0aG9uMwoiIiIKYXNrLWFubmllL2luZ2VzdC5weQoKR2l2ZW4gYSBWaW1lbyBVUkwgYW5kIGEgY2hhcHRlciBsaXN0IChKU09OIGZpbGUpLCBwcm9kdWNlcyBzdHJ1Y3R1cmVkCnBlci1jaGFwdGVyIGtub3dsZWRnZSBjaHVua3Mgc3VpdGFibGUgZm9yIGluZ2VzdGlvbiBpbnRvIGtub3dsZWRnZS1tY3AuCgpVc2FnZToKICBweXRob24zIGluZ2VzdC5weSAtLXVybCA8dmltZW9fdXJsPiAtLWNoYXB0ZXJzIGNoYXB0ZXJzLmpzb24gLS1vdXQgb3V0LyBbLS1mcmFtZXNdCgpPdXRwdXQ6CiAgb3V0Lzx2aWRlb19pZD4vY2h1bmtzLmpzb24gICDigJQgYXJyYXkgb2YgY2hhcHRlciBjaHVua3MKICBvdXQvPHZpZGVvX2lkPi9mcmFtZXMvICAgICAgIOKAlCBleHRyYWN0ZWQgZnJhbWUgaW1hZ2VzIChpZiAtLWZyYW1lcykKICBvdXQvPHZpZGVvX2lkPi90cmFuc2NyaXB0Lmpzb24g4oCUIGZ1bGwgV2hpc3BlciBvdXRwdXQgKGNhY2hlZCkKCkRlcGVuZGVuY2llcyAobXVzdCBiZSBvbiBQQVRIKToKICB5dC1kbHAsIGZmbXBlZywgd2hpc3BlcgoiIiIKCmltcG9ydCBhcmdwYXJzZQppbXBvcnQganNvbgppbXBvcnQgb3MKaW1wb3J0IHN1YnByb2Nlc3MKaW1wb3J0IHN5cwppbXBvcnQgdGVtcGZpbGUKZnJvbSBwYXRobGliIGltcG9ydCBQYXRoCgoKIyBDaGFwdGVycyB3aGVyZSBmcmFtZSBleHRyYWN0aW9uIGFkZHMgcmVhbCB2YWx1ZSAoZGVtby1oZWF2eSBzZWN0aW9ucykuCiMgSWRlbnRpZmllZCBieSBtYXRjaGluZyBjaGFwdGVyIHRpdGxlIHN1YnN0cmluZ3MgKGNhc2UtaW5zZW5zaXRpdmUpLgpERU1PX0NIQVBURVJfS0VZV09SRFMgPSBbCiAgICAidHJhbnNjb2RpbmciLAogICAgInMzIiwKICAgICJmaWxlIHRyYWNraW5nIiwKICAgICJ3b3JrYmVuY2giLAogICAgIm5ldyBjYXBhYmlsaXR5IiwKICAgICJwcmV2aWV3IiwKICAgICJkZW1vIiwKICAgICJ1aSIsCiAgICAic2V0dXAiLApdCgoKZGVmIHJ1bihjbWQsICoqa3dhcmdzKToKICAgIHByaW50KGYiICAkIHsnICcuam9pbihjbWQpfSIsIGZsdXNoPVRydWUpCiAgICByZXN1bHQgPSBzdWJwcm9jZXNzLnJ1bihjbWQsIGNoZWNrPVRydWUsICoqa3dhcmdzKQogICAgcmV0dXJuIHJlc3VsdAoKCmRlZiBzZWNvbmRzX3RvX2hobW1zcyhzKToKICAgIHMgPSBpbnQocykKICAgIGgsIHJlbSA9IGRpdm1vZChzLCAzNjAwKQogICAgbSwgc2VjID0gZGl2bW9kKHJlbSwgNjApCiAgICByZXR1cm4gZiJ7aDowMmR9OnttOjAyZH06e3NlYzowMmR9IgoKCmRlZiBwYXJzZV90aW1lc3RhbXAodHNfc3RyKToKICAgICIiIlBhcnNlICdNOlNTJyBvciAnSDpNTTpTUycgdG8gc2Vjb25kcy4iIiIKICAgIHBhcnRzID0gdHNfc3RyLnN0cmlwKCkuc3BsaXQoIjoiKQogICAgcGFydHMgPSBbaW50KHApIGZvciBwIGluIHBhcnRzXQogICAgaWYgbGVuKHBhcnRzKSA9PSAyOgogICAgICAgIHJldHVybiBwYXJ0c1swXSAqIDYwICsgcGFydHNbMV0KICAgIGVsaWYgbGVuKHBhcnRzKSA9PSAzOgogICAgICAgIHJldHVybiBwYXJ0c1swXSAqIDM2MDAgKyBwYXJ0c1sxXSAqIDYwICsgcGFydHNbMl0KICAgIHJhaXNlIFZhbHVlRXJyb3IoZiJDYW5ub3QgcGFyc2UgdGltZXN0YW1wOiB7dHNfc3RyfSIpCgoKZGVmIGRvd25sb2FkX2F1ZGlvKHVybCwgb3V0X2Rpcik6CiAgICBhdWRpb19wYXRoID0gb3V0X2RpciAvICJhdWRpby4lKGV4dClzIgogICAgcnVuKFsKICAgICAgICAieXQtZGxwIiwKICAgICAgICAiLS1leHRyYWN0LWF1ZGlvIiwKICAgICAgICAiLS1hdWRpby1mb3JtYXQiLCAibXAzIiwKICAgICAgICAiLS1hdWRpby1xdWFsaXR5IiwgIjMiLCAgIyB+MTI4a2JwcywgZ29vZCBlbm91Z2ggZm9yIHNwZWVjaAogICAgICAgICItbyIsIHN0cihhdWRpb19wYXRoKSwKICAgICAgICB1cmwsCiAgICBdKQogICAgIyBGaW5kIHRoZSBkb3dubG9hZGVkIGZpbGUKICAgIG1hdGNoZXMgPSBsaXN0KG91dF9kaXIuZ2xvYigiYXVkaW8uKiIpKQogICAgaWYgbm90IG1hdGNoZXM6CiAgICAgICAgcmFpc2UgRmlsZU5vdEZvdW5kRXJyb3IoIkF1ZGlvIGRvd25sb2FkIGZhaWxlZCDigJQgbm8gYXVkaW8uKiBmaWxlIGZvdW5kIikKICAgIHJldHVybiBtYXRjaGVzWzBdCgoKZGVmIHRyYW5zY3JpYmUoYXVkaW9fcGF0aCwgb3V0X2RpciwgbW9kZWw9Im1lZGl1bSIpOgogICAgdHJhbnNjcmlwdF9wYXRoID0gb3V0X2RpciAvICJ0cmFuc2NyaXB0Lmpzb24iCiAgICBpZiB0cmFuc2NyaXB0X3BhdGguZXhpc3RzKCk6CiAgICAgICAgcHJpbnQoZiIgIFtjYWNoZV0gVXNpbmcgZXhpc3RpbmcgdHJhbnNjcmlwdCBhdCB7dHJhbnNjcmlwdF9wYXRofSIpCiAgICAgICAgd2l0aCBvcGVuKHRyYW5zY3JpcHRfcGF0aCkgYXMgZjoKICAgICAgICAgICAgcmV0dXJuIGpzb24ubG9hZChmKQoKICAgIHJ1bihbCiAgICAgICAgIndoaXNwZXIiLAogICAgICAgIHN0cihhdWRpb19wYXRoKSwKICAgICAgICAiLS1tb2RlbCIsIG1vZGVsLAogICAgICAgICItLW91dHB1dF9mb3JtYXQiLCAianNvbiIsCiAgICAgICAgIi0tb3V0cHV0X2RpciIsIHN0cihvdXRfZGlyKSwKICAgICAgICAiLS1sYW5ndWFnZSIsICJlbiIsCiAgICAgICAgIi0td29yZF90aW1lc3RhbXBzIiwgIlRydWUiLAogICAgXSkKCiAgICAjIFdoaXNwZXIgbmFtZXMgb3V0cHV0IGFmdGVyIHRoZSBpbnB1dCBmaWxlbmFtZQogICAgd2hpc3Blcl9vdXQgPSBvdXRfZGlyIC8gKGF1ZGlvX3BhdGguc3RlbSArICIuanNvbiIpCiAgICBpZiBub3Qgd2hpc3Blcl9vdXQuZXhpc3RzKCk6CiAgICAgICAgcmFpc2UgRmlsZU5vdEZvdW5kRXJyb3IoZiJFeHBlY3RlZCBXaGlzcGVyIG91dHB1dCBhdCB7d2hpc3Blcl9vdXR9IikKICAgIHdoaXNwZXJfb3V0LnJlbmFtZSh0cmFuc2NyaXB0X3BhdGgpCgogICAgd2l0aCBvcGVuKHRyYW5zY3JpcHRfcGF0aCkgYXMgZjoKICAgICAgICByZXR1cm4ganNvbi5sb2FkKGYpCgoKZGVmIGV4dHJhY3RfZnJhbWUodmlkZW9fcGF0aCwgdGltZXN0YW1wX3NlYywgb3V0X3BhdGgpOgogICAgcnVuKFsKICAgICAgICAiZmZtcGVnIiwgIi15IiwKICAgICAgICAiLXNzIiwgc3RyKHRpbWVzdGFtcF9zZWMpLAogICAgICAgICItaSIsIHN0cih2aWRlb19wYXRoKSwKICAgICAgICAiLWZyYW1lczp2IiwgIjEiLAogICAgICAgICItcTp2IiwgIjIiLAogICAgICAgIHN0cihvdXRfcGF0aCksCiAgICBdLCBzdGRvdXQ9c3VicHJvY2Vzcy5ERVZOVUxMLCBzdGRlcnI9c3VicHJvY2Vzcy5ERVZOVUxMKQoKCmRlZiBpc19kZW1vX2NoYXB0ZXIodGl0bGUpOgogICAgdCA9IHRpdGxlLmxvd2VyKCkKICAgIHJldHVybiBhbnkoa3cgaW4gdCBmb3Iga3cgaW4gREVNT19DSEFQVEVSX0tFWVdPUkRTKQoKCmRlZiBzZWdtZW50c19pbl93aW5kb3coc2VnbWVudHMsIHN0YXJ0X3NlYywgZW5kX3NlYyk6CiAgICAiIiJFeHRyYWN0IHRyYW5zY3JpcHQgdGV4dCBmb3IgYSB0aW1lIHdpbmRvdyBmcm9tIFdoaXNwZXIgc2VnbWVudHMuIiIiCiAgICB0ZXh0cyA9IFtdCiAgICBmb3Igc2VnIGluIHNlZ21lbnRzOgogICAgICAgIHNlZ19zdGFydCA9IHNlZy5nZXQoInN0YXJ0IiwgMCkKICAgICAgICBzZWdfZW5kID0gc2VnLmdldCgiZW5kIiwgc2VnX3N0YXJ0KQogICAgICAgICMgSW5jbHVkZSBzZWdtZW50IGlmIGl0IG92ZXJsYXBzIHdpdGggdGhlIHdpbmRvdwogICAgICAgIGlmIHNlZ19lbmQgPj0gc3RhcnRfc2VjIGFuZCBzZWdfc3RhcnQgPD0gZW5kX3NlYzoKICAgICAgICAgICAgdGV4dHMuYXBwZW5kKHNlZ1sidGV4dCJdLnN0cmlwKCkpCiAgICByZXR1cm4gIiAiLmpvaW4odGV4dHMpCgoKZGVmIGRlc2NyaWJlX2ZyYW1lKGltYWdlX3BhdGgpOgogICAgIiIiCiAgICBQbGFjZWhvbGRlcjogcmV0dXJucyBhIG5vdGUgdGhhdCBmcmFtZSBkZXNjcmlwdGlvbiBuZWVkcyBDbGF1ZGUgdmlzaW9uLgogICAgSW4gYSBsYXRlciBwYXNzLCByZXBsYWNlIHRoaXMgd2l0aCBhbiBhY3R1YWwgQVBJIGNhbGwgb3Igb3V0cHV0IHRoZQogICAgaW1hZ2UgcGF0aCBmb3IgZXh0ZXJuYWwgcHJvY2Vzc2luZy4KICAgICIiIgogICAgcmV0dXJuIGYiW2ZyYW1lOiB7aW1hZ2VfcGF0aC5uYW1lfSDigJQgdmlzaW9uIGRlc2NyaXB0aW9uIHBlbmRpbmddIgoKCmRlZiBtYWluKCk6CiAgICBwYXJzZXIgPSBhcmdwYXJzZS5Bcmd1bWVudFBhcnNlcihkZXNjcmlwdGlvbj0iSW5nZXN0IEFzayBBbm5pZSBWaW1lbyBzZXNzaW9uIikKICAgIHBhcnNlci5hZGRfYXJndW1lbnQoIi0tdXJsIiwgcmVxdWlyZWQ9VHJ1ZSwgaGVscD0iVmltZW8gVVJMIikKICAgIHBhcnNlci5hZGRfYXJndW1lbnQoIi0tY2hhcHRlcnMiLCByZXF1aXJlZD1UcnVlLCBoZWxwPSJQYXRoIHRvIGNoYXB0ZXJzIEpTT04gZmlsZSIpCiAgICBwYXJzZXIuYWRkX2FyZ3VtZW50KCItLW91dCIsIGRlZmF1bHQ9Im91dCIsIGhlbHA9Ik91dHB1dCBkaXJlY3RvcnkiKQogICAgcGFyc2VyLmFkZF9hcmd1bWVudCgiLS1mcmFtZXMiLCBhY3Rpb249InN0b3JlX3RydWUiLCBoZWxwPSJFeHRyYWN0IGZyYW1lcyBmb3IgZGVtbyBjaGFwdGVycyIpCiAgICBwYXJzZXIuYWRkX2FyZ3VtZW50KCItLXdoaXNwZXItbW9kZWwiLCBkZWZhdWx0PSJtZWRpdW0iLCBoZWxwPSJXaGlzcGVyIG1vZGVsIHNpemUiKQogICAgcGFyc2VyLmFkZF9hcmd1bWVudCgiLS12aWRlby1pZCIsIGRlZmF1bHQ9Tm9uZSwgaGVscD0iT3ZlcnJpZGUgdmlkZW8gSUQgKGV4dHJhY3RlZCBmcm9tIFVSTCBpZiBvbWl0dGVkKSIpCiAgICBhcmdzID0gcGFyc2VyLnBhcnNlX2FyZ3MoKQoKICAgICMgRGVyaXZlIHZpZGVvIElEIGZyb20gVVJMCiAgICB2aWRlb19pZCA9IGFyZ3MudmlkZW9faWQgb3IgYXJncy51cmwucnN0cmlwKCIvIikuc3BsaXQoIi8iKVstMV0uc3BsaXQoIj8iKVswXQogICAgcHJpbnQoZiJcbj09PSBBc2sgQW5uaWUgSW5nZXN0OiB7dmlkZW9faWR9ID09PVxuIikKCiAgICBvdXRfZGlyID0gUGF0aChhcmdzLm91dCkgLyB2aWRlb19pZAogICAgZnJhbWVzX2RpciA9IG91dF9kaXIgLyAiZnJhbWVzIgogICAgb3V0X2Rpci5ta2RpcihwYXJlbnRzPVRydWUsIGV4aXN0X29rPVRydWUpCiAgICBpZiBhcmdzLmZyYW1lczoKICAgICAgICBmcmFtZXNfZGlyLm1rZGlyKGV4aXN0X29rPVRydWUpCgogICAgIyBMb2FkIGNoYXB0ZXJzCiAgICB3aXRoIG9wZW4oYXJncy5jaGFwdGVycykgYXMgZjoKICAgICAgICBjaGFwdGVycyA9IGpzb24ubG9hZChmKQogICAgcHJpbnQoZiJMb2FkZWQge2xlbihjaGFwdGVycyl9IGNoYXB0ZXJzXG4iKQoKICAgICMgU3RlcCAxOiBEb3dubG9hZCBhdWRpbwogICAgcHJpbnQoIj09PSBTdGVwIDE6IERvd25sb2FkIGF1ZGlvID09PSIpCiAgICBhdWRpb19wYXRoID0gTm9uZQogICAgZm9yIGYgaW4gb3V0X2Rpci5nbG9iKCJhdWRpby4qIik6CiAgICAgICAgcHJpbnQoZiIgIFtjYWNoZV0gRm91bmQgZXhpc3RpbmcgYXVkaW86IHtmfSIpCiAgICAgICAgYXVkaW9fcGF0aCA9IGYKICAgICAgICBicmVhawogICAgaWYgYXVkaW9fcGF0aCBpcyBOb25lOgogICAgICAgIGF1ZGlvX3BhdGggPSBkb3dubG9hZF9hdWRpbyhhcmdzLnVybCwgb3V0X2RpcikKICAgIHByaW50KGYiICBBdWRpbzoge2F1ZGlvX3BhdGh9XG4iKQoKICAgICMgU3RlcCAxYjogRG93bmxvYWQgdmlkZW8gKG9ubHkgaWYgLS1mcmFtZXMgcmVxdWVzdGVkKQogICAgdmlkZW9fcGF0aCA9IE5vbmUKICAgIGlmIGFyZ3MuZnJhbWVzOgogICAgICAgIHByaW50KCI9PT0gU3RlcCAxYjogRG93bmxvYWQgdmlkZW8gKGZvciBmcmFtZSBleHRyYWN0aW9uKSA9PT0iKQogICAgICAgIGV4aXN0aW5nID0gbGlzdChvdXRfZGlyLmdsb2IoInZpZGVvLioiKSkKICAgICAgICBpZiBleGlzdGluZzoKICAgICAgICAgICAgdmlkZW9fcGF0aCA9IGV4aXN0aW5nWzBdCiAgICAgICAgICAgIHByaW50KGYiICBbY2FjaGVdIEZvdW5kIGV4aXN0aW5nIHZpZGVvOiB7dmlkZW9fcGF0aH0iKQogICAgICAgIGVsc2U6CiAgICAgICAgICAgIHZpZF9vdXQgPSBvdXRfZGlyIC8gInZpZGVvLiUoZXh0KXMiCiAgICAgICAgICAgIHJ1bihbInl0LWRscCIsICItZiIsICJiZXN0dmlkZW9baGVpZ2h0PD03MjBdIiwgIi1vIiwgc3RyKHZpZF9vdXQpLCBhcmdzLnVybF0pCiAgICAgICAgICAgIG1hdGNoZXMgPSBsaXN0KG91dF9kaXIuZ2xvYigidmlkZW8uKiIpKQogICAgICAgICAgICBpZiBub3QgbWF0Y2hlczoKICAgICAgICAgICAgICAgIHByaW50KCIgIFdBUk5JTkc6IFZpZGVvIGRvd25sb2FkIGZhaWxlZCwgc2tpcHBpbmcgZnJhbWUgZXh0cmFjdGlvbiIpCiAgICAgICAgICAgICAgICBhcmdzLmZyYW1lcyA9IEZhbHNlCiAgICAgICAgICAgIGVsc2U6CiAgICAgICAgICAgICAgICB2aWRlb19wYXRoID0gbWF0Y2hlc1swXQogICAgICAgIHByaW50KCkKCiAgICAjIFN0ZXAgMjogVHJhbnNjcmliZQogICAgcHJpbnQoIj09PSBTdGVwIDI6IFRyYW5zY3JpYmUgPT09IikKICAgIHRyYW5zY3JpcHQgPSB0cmFuc2NyaWJlKGF1ZGlvX3BhdGgsIG91dF9kaXIsIG1vZGVsPWFyZ3Mud2hpc3Blcl9tb2RlbCkKICAgIHNlZ21lbnRzID0gdHJhbnNjcmlwdC5nZXQoInNlZ21lbnRzIiwgW10pCiAgICBwcmludChmIiAgR290IHtsZW4oc2VnbWVudHMpfSB0cmFuc2NyaXB0IHNlZ21lbnRzXG4iKQoKICAgICMgU3RlcCAzOiBCdWlsZCBjaHVua3MKICAgIHByaW50KCI9PT0gU3RlcCAzOiBCdWlsZCBjaHVua3MgPT09IikKICAgIGNodW5rcyA9IFtdCiAgICBmb3IgaSwgY2hhcHRlciBpbiBlbnVtZXJhdGUoY2hhcHRlcnMpOgogICAgICAgIHN0YXJ0X3NlYyA9IHBhcnNlX3RpbWVzdGFtcChjaGFwdGVyWyJ0aW1lc3RhbXAiXSkKICAgICAgICAjIEVuZCA9IG5leHQgY2hhcHRlciBzdGFydCwgb3IgKzEwbWluIGZvciBsYXN0CiAgICAgICAgaWYgaSArIDEgPCBsZW4oY2hhcHRlcnMpOgogICAgICAgICAgICBlbmRfc2VjID0gcGFyc2VfdGltZXN0YW1wKGNoYXB0ZXJzW2kgKyAxXVsidGltZXN0YW1wIl0pCiAgICAgICAgZWxzZToKICAgICAgICAgICAgZW5kX3NlYyA9IHN0YXJ0X3NlYyArIDYwMAoKICAgICAgICB0ZXh0ID0gc2VnbWVudHNfaW5fd2luZG93KHNlZ21lbnRzLCBzdGFydF9zZWMsIGVuZF9zZWMpCiAgICAgICAgZGVtbyA9IGlzX2RlbW9fY2hhcHRlcihjaGFwdGVyWyJ0aXRsZSJdKQoKICAgICAgICBmcmFtZV9kZXNjID0gTm9uZQogICAgICAgIGlmIGFyZ3MuZnJhbWVzIGFuZCBkZW1vIGFuZCB2aWRlb19wYXRoOgogICAgICAgICAgICBmcmFtZV9maWxlID0gZnJhbWVzX2RpciAvIGYiY2hhcHRlcl97aTowMmR9LmpwZyIKICAgICAgICAgICAgcHJpbnQoZiIgIEV4dHJhY3RpbmcgZnJhbWUgZm9yOiB7Y2hhcHRlclsndGl0bGUnXX0iKQogICAgICAgICAgICBleHRyYWN0X2ZyYW1lKHZpZGVvX3BhdGgsIHN0YXJ0X3NlYyArIDUsIGZyYW1lX2ZpbGUpCiAgICAgICAgICAgIGZyYW1lX2Rlc2MgPSBkZXNjcmliZV9mcmFtZShmcmFtZV9maWxlKQoKICAgICAgICBjaHVuayA9IHsKICAgICAgICAgICAgInZpZGVvX2lkIjogdmlkZW9faWQsCiAgICAgICAgICAgICJ2aWRlb191cmwiOiBhcmdzLnVybCwKICAgICAgICAgICAgImNoYXB0ZXJfaW5kZXgiOiBpLAogICAgICAgICAgICAidGltZXN0YW1wIjogY2hhcHRlclsidGltZXN0YW1wIl0sCiAgICAgICAgICAgICJ0aW1lc3RhbXBfc2VjIjogc3RhcnRfc2VjLAogICAgICAgICAgICAidGl0bGUiOiBjaGFwdGVyWyJ0aXRsZSJdLAogICAgICAgICAgICAic3VtbWFyeSI6IGNoYXB0ZXIuZ2V0KCJzdW1tYXJ5IiwgIiIpLAogICAgICAgICAgICAidHJhbnNjcmlwdCI6IHRleHQsCiAgICAgICAgICAgICJpc19kZW1vIjogZGVtbywKICAgICAgICAgICAgImZyYW1lX2Rlc2NyaXB0aW9uIjogZnJhbWVfZGVzYywKICAgICAgICAgICAgInNvdXJjZSI6ICJhc2stYW5uaWUiLAogICAgICAgICAgICAic2VyaWVzIjogIlNUIEJlc3QgUHJhY3RpY2VzIFEmQSIsCiAgICAgICAgfQogICAgICAgIGNodW5rcy5hcHBlbmQoY2h1bmspCiAgICAgICAgcHJpbnQoZiIgIFt7aTowMmR9XSB7Y2hhcHRlclsndGltZXN0YW1wJ119IOKAlCB7Y2hhcHRlclsndGl0bGUnXVs6NjBdfSAoeydkZW1vJyBpZiBkZW1vIGVsc2UgJ3FhJ30pIikKCiAgICAjIFN0ZXAgNDogV3JpdGUgb3V0cHV0CiAgICBjaHVua3NfcGF0aCA9IG91dF9kaXIgLyAiY2h1bmtzLmpzb24iCiAgICB3aXRoIG9wZW4oY2h1bmtzX3BhdGgsICJ3IikgYXMgZjoKICAgICAgICBqc29uLmR1bXAoY2h1bmtzLCBmLCBpbmRlbnQ9MikKICAgIHByaW50KGYiXG49PT0gRG9uZToge2xlbihjaHVua3MpfSBjaHVua3Mg4oaSIHtjaHVua3NfcGF0aH0gPT09XG4iKQoKCmlmIF9fbmFtZV9fID09ICJfX21haW5fXyI6CiAgICBtYWluKCkK \ No newline at end of file +#!/usr/bin/env python3 +""" +ask-annie/ingest.py + +Given a Vimeo URL and a chapter list (JSON file), produces structured +per-chapter knowledge chunks suitable for ingestion into knowledge-mcp. + +Usage: + python3 ingest.py --url --chapters chapters.json --out out/ [--frames] + +Output: + out//chunks.json — array of chapter chunks + out//frames/ — extracted frame images (if --frames) + out//transcript.json — full Whisper output (cached) + +Dependencies (must be on PATH): + yt-dlp, ffmpeg, whisper +""" + +import argparse +import json +import os +import subprocess +import sys +import tempfile +from pathlib import Path + + +# Chapters where frame extraction adds real value (demo-heavy sections). +# Identified by matching chapter title substrings (case-insensitive). +DEMO_CHAPTER_KEYWORDS = [ + "transcoding", + "s3", + "file tracking", + "workbench", + "new capability", + "preview", + "demo", + "ui", + "setup", +] + + +def run(cmd, **kwargs): + print(f" $ {' '.join(cmd)}", flush=True) + result = subprocess.run(cmd, check=True, **kwargs) + return result + + +def seconds_to_hhmmss(s): + s = int(s) + h, rem = divmod(s, 3600) + m, sec = divmod(rem, 60) + return f"{h:02d}:{m:02d}:{sec:02d}" + + +def parse_timestamp(ts_str): + """Parse 'M:SS' or 'H:MM:SS' to seconds.""" + parts = ts_str.strip().split(":") + parts = [int(p) for p in parts] + if len(parts) == 2: + return parts[0] * 60 + parts[1] + elif len(parts) == 3: + return parts[0] * 3600 + parts[1] * 60 + parts[2] + raise ValueError(f"Cannot parse timestamp: {ts_str}") + + +def download_audio(url, out_dir): + audio_path = out_dir / "audio.%(ext)s" + run([ + "yt-dlp", + "--extract-audio", + "--audio-format", "mp3", + "--audio-quality", "3", # ~128kbps, good enough for speech + "-o", str(audio_path), + url, + ]) + # Find the downloaded file + matches = list(out_dir.glob("audio.*")) + if not matches: + raise FileNotFoundError("Audio download failed — no audio.* file found") + return matches[0] + + +def transcribe(audio_path, out_dir, model="medium"): + transcript_path = out_dir / "transcript.json" + if transcript_path.exists(): + print(f" [cache] Using existing transcript at {transcript_path}") + with open(transcript_path) as f: + return json.load(f) + + run([ + "whisper", + str(audio_path), + "--model", model, + "--output_format", "json", + "--output_dir", str(out_dir), + "--language", "en", + "--word_timestamps", "True", + ]) + + # Whisper names output after the input filename + whisper_out = out_dir / (audio_path.stem + ".json") + if not whisper_out.exists(): + raise FileNotFoundError(f"Expected Whisper output at {whisper_out}") + whisper_out.rename(transcript_path) + + with open(transcript_path) as f: + return json.load(f) + + +def extract_frame(video_path, timestamp_sec, out_path): + run([ + "ffmpeg", "-y", + "-ss", str(timestamp_sec), + "-i", str(video_path), + "-frames:v", "1", + "-q:v", "2", + str(out_path), + ], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + + +def is_demo_chapter(title): + t = title.lower() + return any(kw in t for kw in DEMO_CHAPTER_KEYWORDS) + + +def segments_in_window(segments, start_sec, end_sec): + """Extract transcript text for a time window from Whisper segments.""" + texts = [] + for seg in segments: + seg_start = seg.get("start", 0) + seg_end = seg.get("end", seg_start) + # Include segment if it overlaps with the window + if seg_end >= start_sec and seg_start <= end_sec: + texts.append(seg["text"].strip()) + return " ".join(texts) + + +def describe_frame(image_path): + """ + Placeholder: returns a note that frame description needs Claude vision. + In a later pass, replace this with an actual API call or output the + image path for external processing. + """ + return f"[frame: {image_path.name} — vision description pending]" + + +def main(): + parser = argparse.ArgumentParser(description="Ingest Ask Annie Vimeo session") + parser.add_argument("--url", required=True, help="Vimeo URL") + parser.add_argument("--chapters", required=True, help="Path to chapters JSON file") + parser.add_argument("--out", default="out", help="Output directory") + parser.add_argument("--frames", action="store_true", help="Extract frames for demo chapters") + parser.add_argument("--whisper-model", default="medium", help="Whisper model size") + parser.add_argument("--video-id", default=None, help="Override video ID (extracted from URL if omitted)") + args = parser.parse_args() + + # Derive video ID from URL + video_id = args.video_id or args.url.rstrip("/").split("/")[-1].split("?")[0] + print(f"\n=== Ask Annie Ingest: {video_id} ===\n") + + out_dir = Path(args.out) / video_id + frames_dir = out_dir / "frames" + out_dir.mkdir(parents=True, exist_ok=True) + if args.frames: + frames_dir.mkdir(exist_ok=True) + + # Load chapters + with open(args.chapters) as f: + chapters = json.load(f) + print(f"Loaded {len(chapters)} chapters\n") + + # Step 1: Download audio + print("=== Step 1: Download audio ===") + audio_path = None + for f in out_dir.glob("audio.*"): + print(f" [cache] Found existing audio: {f}") + audio_path = f + break + if audio_path is None: + audio_path = download_audio(args.url, out_dir) + print(f" Audio: {audio_path}\n") + + # Step 1b: Download video (only if --frames requested) + video_path = None + if args.frames: + print("=== Step 1b: Download video (for frame extraction) ===") + existing = list(out_dir.glob("video.*")) + if existing: + video_path = existing[0] + print(f" [cache] Found existing video: {video_path}") + else: + vid_out = out_dir / "video.%(ext)s" + run(["yt-dlp", "-f", "bestvideo[height<=720]", "-o", str(vid_out), args.url]) + matches = list(out_dir.glob("video.*")) + if not matches: + print(" WARNING: Video download failed, skipping frame extraction") + args.frames = False + else: + video_path = matches[0] + print() + + # Step 2: Transcribe + print("=== Step 2: Transcribe ===") + transcript = transcribe(audio_path, out_dir, model=args.whisper_model) + segments = transcript.get("segments", []) + print(f" Got {len(segments)} transcript segments\n") + + # Step 3: Build chunks + print("=== Step 3: Build chunks ===") + chunks = [] + for i, chapter in enumerate(chapters): + start_sec = parse_timestamp(chapter["timestamp"]) + # End = next chapter start, or +10min for last + if i + 1 < len(chapters): + end_sec = parse_timestamp(chapters[i + 1]["timestamp"]) + else: + end_sec = start_sec + 600 + + text = segments_in_window(segments, start_sec, end_sec) + demo = is_demo_chapter(chapter["title"]) + + frame_desc = None + if args.frames and demo and video_path: + frame_file = frames_dir / f"chapter_{i:02d}.jpg" + print(f" Extracting frame for: {chapter['title']}") + extract_frame(video_path, start_sec + 5, frame_file) + frame_desc = describe_frame(frame_file) + + chunk = { + "video_id": video_id, + "video_url": args.url, + "chapter_index": i, + "timestamp": chapter["timestamp"], + "timestamp_sec": start_sec, + "title": chapter["title"], + "summary": chapter.get("summary", ""), + "transcript": text, + "is_demo": demo, + "frame_description": frame_desc, + "source": "ask-annie", + "series": "ST Best Practices Q&A", + } + chunks.append(chunk) + print(f" [{i:02d}] {chapter['timestamp']} — {chapter['title'][:60]} ({'demo' if demo else 'qa'})") + + # Step 4: Write output + chunks_path = out_dir / "chunks.json" + with open(chunks_path, "w") as f: + json.dump(chunks, f, indent=2) + print(f"\n=== Done: {len(chunks)} chunks → {chunks_path} ===\n") + + +if __name__ == "__main__": + main()