fixes and adjustments to avoid rate-limit and other weird notion issues

Author: dotansimha

README.md

@@ -66,6 +66,7 @@ The following configurations can be set in the env of your project, in order to
 - `DRY_RUN` - set to `1` if you wish to just test the create/update/delete plan of this bot, without affecting any data on GitHub.
 - `ENABLE_FETCH` - Set to `1` to enable. This will enable the `fetch` event for the worker, this is helpful for development if you want to trigger the bot manually, or if you wish your bot to have a manual trigger.
 - `CUSTOM_HEADER_LINK` - customize the link added to the header of every GitHub issue/discussion. To use an external like, you can add markdown, for example: `[The Guild's](https://the-guild.dev)`.
+- `IGNORED_REPOS` - If the GitHub user you are using, is used for creating other issues/discussions, or a real (non-bot) user, you can reduce the stress of loading existing issues/discussion with ignoring some repos. This field is comma-separated (for example: `user/repo1,user/repo2`)
 
 For local development, please add your config to a file called `.dev.vars`
 

src/index.ts

@@ -21,9 +21,13 @@ export interface Env {
   DRY_RUN?: string;
   ENABLE_FETCH?: string;
   CUSTOM_HEADER_LINK?: string;
+  IGNORED_REPOS?: string;
 }
 
 async function run(env: Env) {
+  const ignoredRepos = (env.IGNORED_REPOS || "")
+    .split(",")
+    .map((v) => v.trim());
   const shouldExecute = !env.DRY_RUN;
   const botBrand = env.CUSTOM_HEADER_LINK ? `${env.CUSTOM_HEADER_LINK} ` : "";
   const headerNote = `> This page is synced automatically from ${botBrand}Notion`;
@@ -34,12 +38,15 @@ async function run(env: Env) {
   const n2m = new NotionToMarkdown({ notionClient: notion });
   const octokit = new Octokit({ auth: env.GH_BOT_TOKEN });
   const login = await getBotLogin(octokit);
+  console.info(`GitHub user identified as: ${login}`);
   const [relevantPages, discussions, issues] = await Promise.all([
     getSharedNotionPages(notion),
-    getExistingDiscussions(octokit, login),
-    getExistingIssues(octokit, login),
+    getExistingDiscussions(octokit, login, ignoredRepos),
+    getExistingIssues(octokit, login, ignoredRepos),
   ]);
-  console.log("existing issues found:", issues);
+  console.info("Found existing issues:", issues);
+  console.info("Found existing discussions:", discussions);
+  console.info("Found shared Notion pages:", relevantPages);
   const { discussions: discussionsPlan, issues: issuesPlan } =
     await buildUpdatePlan(
       octokit,
@@ -50,8 +57,8 @@ async function run(env: Env) {
       headerNote
     );
 
-  console.info(`Built Discussion sync plan:`, discussionsPlan);
-  console.info(`Built Issues sync plan:`, issuesPlan);
+  console.info(`Built GitHub Discussion sync plan:`, discussionsPlan);
+  console.info(`Built GitHub Issues sync plan:`, issuesPlan);
 
   await Promise.all([
     ...discussionsPlan.delete.map(async (item) => {

src/notion-helpers.ts

@@ -1,4 +1,4 @@
-import { Client } from "@notionhq/client";
+import { Client, isFullPage } from "@notionhq/client";
 import { Page } from "./utils";
 
 export function composeLink(page: Page): string {
@@ -38,7 +38,35 @@ export function extractPageTitle(page: Page): string | null {
 export async function getSharedNotionPages(notion: Client) {
   const relevantPages = await notion.search({
     page_size: 100,
+    filter: {
+      property: "object",
+      value: "page",
+    },
+    sort: {
+      direction: "descending",
+      timestamp: "last_edited_time",
+    },
   });
 
   return relevantPages.results;
 }
+
+export function shouldHandlePage(page: any): boolean {
+  if (isFullPage(page)) {
+    if (page.archived) {
+      return false;
+    }
+
+    // These are usually created by Notion when you create a DB inside Page.
+    // In most cases, these pages are empty and causes the bot to overfetch, until it gets rate limited.
+    const isMultiSelect = page.properties?.Type?.type === "multi_select";
+
+    if (isMultiSelect) {
+      return false;
+    }
+
+    return true;
+  }
+
+  return false;
+}

src/plan.ts

@@ -5,6 +5,7 @@ import {
   composeLink,
   composeSignature,
   extractPageTitle,
+  shouldHandlePage,
 } from "./notion-helpers";
 import {
   Discussion,
@@ -107,10 +108,21 @@ export async function buildUpdatePlan(
       const pageTitle = extractPageTitle(page);
 
       if (!pageTitle) {
+        console.warn(`Notion object with id ${page.id} has no title, skipping`);
+
+        return;
+      }
+
+      const shouldHandle = shouldHandlePage(page);
+
+      if (!shouldHandle) {
+        console.warn(
+          `Notion object with title ${pageTitle} is not a syncable page, skipping`
+        );
+
         return;
       }
 
-      console.info(`Building plan for page: `, pageTitle, page);
       const mdBlocks = await n2m.pageToMarkdown(page.id, 2);
       const pageAttributes = distinguishPage(mdBlocks[0]);
       const notionPageSignature = composeSignature(page.id);
@@ -122,6 +134,8 @@ export async function buildUpdatePlan(
       );
 
       if (pageAttributes === null) {
+        console.warn(`Notion page with title "${pageTitle}" has no attributes`);
+
         if (existingDiscussion) {
           outputDiscussions.delete.push({
             repoId: existingDiscussion.repository.id,

src/utils.ts

@@ -31,7 +31,11 @@ function isDiscussion(obj: DiscussionsSearchResult): obj is Discussion {
   return obj?.__typename === "Discussion";
 }
 
-export async function getExistingDiscussions(octokit: Octokit, login: string) {
+export async function getExistingDiscussions(
+  octokit: Octokit,
+  login: string,
+  ignoredRepos: string[] = []
+) {
   const discussionsByBot = await octokit.graphql<MyDiscussionsQuery>(
     /* GraphQL */ `
       query myDiscussions($q: String!) {
@@ -58,7 +62,7 @@ export async function getExistingDiscussions(octokit: Octokit, login: string) {
       }
     `,
     {
-      q: `author:${login} -repo:the-guild-org/crisp-chats`,
+      q: `author:${login} ${ignoredRepos.map((v) => `-repo:${v}`).join(" ")}`,
     }
   );
 
@@ -69,7 +73,11 @@ function isIssue(obj: IssuesSearchResult): obj is Issue {
   return obj?.__typename === "Issue";
 }
 
-export async function getExistingIssues(octokit: Octokit, login: string) {
+export async function getExistingIssues(
+  octokit: Octokit,
+  login: string,
+  ignoredRepos: string[]
+) {
   const issuesByBot = await octokit.graphql<MyIssuesQuery>(
     /* GraphQL */ `
       query myIssues($q: String!) {
@@ -96,7 +104,7 @@ export async function getExistingIssues(octokit: Octokit, login: string) {
       }
     `,
     {
-      q: `author:${login} -repo:the-guild-org/crisp-chats`,
+      q: `author:${login} ${ignoredRepos.map((v) => `-repo:${v}`).join(" ")}`,
     }
   );
 

wrangler.toml

@@ -7,4 +7,5 @@ crons = ["*/5 * * * *"] # every 5 minutes
 
 [vars]
 CUSTOM_HEADER_LINK = "[The Guild's](https://the-guild.dev)"
-ENABLE_FETCH = "1"
+ENABLE_FETCH = "1"
+IGNORED_REPOS = "the-guild-org/crisp-chats"