Import from RSS feeds

Author: fileformat

.editorconfig

@@ -0,0 +1,16 @@
+# EditorConfig is awesome: http://EditorConfig.org
+
+# top-most EditorConfig file
+root = true
+
+[*]
+charset = utf-8
+end_of_line = lf
+indent_size = 4
+indent_style = tab
+insert_final_newline = true
+trim_trailing_whitespace = true
+
+[*.{json,yaml}]
+indent_size = 2
+indent_style = space

.github/workflows/gcr-deploy.yaml

@@ -21,7 +21,7 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v1
+        uses: actions/checkout@v4
 
       - name: gcloud auth
         id: 'auth'

TODO.md

@@ -1,5 +1,9 @@
 # To Do
 
+- [ ] Redos check
+- [ ] Diagram 
+- [ ] Glossary
+- [ ] Tools page w/above
 - [ ] copy to clipboard for variations
 - [ ] patterns into database
 - [ ] search.html
@@ -16,6 +20,21 @@
 - [ ] post to places besides RXP
 - [ ] 404 page
 
+## Patterns
+
+- [ ] URL: https://mathiasbynens.be/demo/url-regex
+- [ ] Semver: https://semver.org/#is-there-a-suggested-regular-expression-regex-to-check-a-semver-string
+- [ ] RFC 3339 dates: https://datatracker.ietf.org/doc/html/rfc3339
+- [ ] RFC 822 dates: https://datatracker.ietf.org/doc/html/rfc822#section-5
+- [ ] IPv4
+- [ ] IPv6
+- [ ] SWIFT
+- [ ] NANP phone number
+- [ ] USA social security numbers
+- [ ] Other national numbers
+- [ ] Other postal codes
+- [ ] 
+
 
 ## Data Model
 

app/routes/links._index.tsx

@@ -57,7 +57,8 @@ export default function Index() {
                     {user && user.isAdmin ?
                         <>
                             <RemixLink to="/links/add.html" className="btn btn-primary mx-1"><AdminIcon /> Add</RemixLink>
-                            <RemixLink to="/links/import.html" className="btn btn-primary mx-1"><AdminIcon /> Import</RemixLink>
+                            <RemixLink to="/links/import-feed.html" className="btn btn-primary mx-1"><AdminIcon /> Import RSS/Adom Feed</RemixLink>
+                            <RemixLink to="/links/import-json.html" className="btn btn-primary mx-1"><AdminIcon /> Import JSON</RemixLink>
                             <a href="/links/backup.json" className="btn btn-primary mx-1"><AdminIcon /> Backup</a>
                         </>
                         : null}

app/routes/links.import-feed[.]html.tsx

@@ -0,0 +1,143 @@
+import {
+    ActionFunctionArgs,
+    LoaderFunctionArgs,
+    MetaFunction,
+    redirect,
+} from "@remix-run/node";
+import { Form, useLoaderData } from "@remix-run/react";
+import { eq } from "drizzle-orm";
+import { parseFeed } from "@rowanmanning/feed-parser";
+
+import { authenticator } from "~/services/auth.server";
+import { adminOnlyLoader } from "~/util/adminOnlyLoader";
+import { User } from "~/types/User";
+
+export const meta: MetaFunction = () => {
+    return [
+        { title: "Import RSS/Atom Links - Regex Zone" },
+        { name: "description", content: "Import JSON link dump from Pinboard.in" },
+    ];
+};
+
+export const action = async ({
+    request,
+}: ActionFunctionArgs) => {
+
+    const user: User | null = await authenticator.isAuthenticated(request);
+    if (!user) {
+        return redirect("/auth/");
+    }
+    if (!user.isAdmin) {
+        throw new Response("Unauthorized", { status: 401 });
+    }
+
+    const formData = await request.formData();
+
+    const formDataValue = formData.get("feedurl");
+    if (!formDataValue) {
+        throw new Response("No url specified", { status: 400 });
+    }
+
+    const feedUrl = formDataValue as string;
+    try {
+        new URL(feedUrl);
+    } catch (e) {
+        throw new Response("Invalid URL", { status: 400 });
+    }
+
+    const feedResponse = await fetch(feedUrl);
+    if (!feedResponse.ok) {
+        throw new Response("Error fetching feed", { status: 400 });
+    }
+
+    const feedtext = await feedResponse.text();
+    let feed:Feed;
+
+    try {
+        feed = await parseFeed(feedtext);
+    } catch (err: unknown) {
+        throw new Response("Error parsing feed", { status: 400 });
+    }
+
+    let count = 0;
+    for (const entry of feed.items) {
+        const url = entry.url;
+        if (!url) {
+            continue;
+        }
+        const existing = await dborm.select().from(regex_link).where(eq(regex_link.rxl_url, url));
+        if (existing.length > 0) {
+            console.log(`found ${url}, stopping...`);
+            break;
+        }
+        const values = {
+            rxl_url: entry.url,
+            rxl_title: entry.title || "(untitled)",
+            rxl_tags: entry.categories[0].term.split(' '),
+            rxl_created_at: new Date(entry.published || "1970-01-01T00:00:00Z"),
+        }
+        console.log(JSON.stringify(values, null, 2));
+        await dborm.insert(regex_link).values(values);
+        count++;
+    }
+
+    const session = await cookieStorage.getSession(request.headers.get("Cookie"));
+    session.flash("message", { type: 'info', message: `You uploaded ${count} entries from ${feedUrl}!` });
+    return redirect('/links/', { headers: { "Set-Cookie": await cookieStorage.commitSession(session) } });
+
+    /**
+     * LATER: it would be nice if Remix supported action responses but `shouldRevalidate` doesn't seem to work
+     *
+    console.log("about to return");
+    return new Response(`You uploaded ${data.length} bytes`, {
+        status: 200,
+        headers: {
+            'Content-type': "text/plain; charset=utf-8",
+        }
+    });
+    */
+
+};
+
+export function loader(args: LoaderFunctionArgs) {
+    const user = adminOnlyLoader(args);
+
+    return {
+        user,
+        url: process.env.BOOKMARK_FEED_URL,
+    }
+}
+
+import type { ShouldRevalidateFunction } from "@remix-run/react";
+import { cookieStorage } from "~/services/session.server";
+import { dborm } from "~/db/connection.server";
+import { regex_link } from "~/db/schema";
+import { Feed } from "@rowanmanning/feed-parser/lib/feed/base";
+
+export const shouldRevalidate: ShouldRevalidateFunction = (params) => {
+    console.log("shouldRevalidate", params);    // never called
+    return params.defaultShouldRevalidate;
+};
+
+export default function Import() {
+
+    const data = useLoaderData<typeof loader>();
+
+    const url = data.url;
+
+
+    return (
+        <>
+            <h1 className="py-2">Import RSS/Atom Feed Links</h1>
+            <div className="d-flex justify-content-center">
+                <Form method="post" className="border p-3" action="/links/import-feed.html" encType="multipart/form-data">
+                    <div className="mb-3">
+                        <label htmlFor="feedurl" className="form-label">RSS/Atom Feed</label>
+                        <input className="form-control" type="url" id="feedurl" name="feedurl" value={url}/>
+                    </div>
+                    <button type="submit" className="btn btn-primary">Import</button>
+                </Form>
+            </div>
+        </>
+    );
+}

app/routes/links.import[.]html.tsx renamed to app/routes/links.import-json[.]html.tsx

@@ -22,7 +22,7 @@ type PinboardEntry = {
 
 export const meta: MetaFunction = () => {
     return [
-        { title: "Import Links - Regex Zone" },
+        { title: "Import JSON Links - Regex Zone" },
         { name: "description", content: "Import JSON link dump from Pinboard.in" },
     ];
 };
@@ -112,7 +112,7 @@ export default function Import() {
         <>
             <h1 className="py-2">Import Links</h1>
             <div className="d-flex justify-content-center">
-                <Form method="post" className="border p-3" action="/links/import.html" encType="multipart/form-data">
+                <Form method="post" className="border p-3" action="/links/import-json.html" encType="multipart/form-data">
                     <div className="mb-3">
                         <label htmlFor="formFile" className="form-label">Pinboard.in Export JSON</label>
                         <input className="form-control" type="file" id="file" name="file" />

package-lock.json

@@ -14,6 +14,7 @@
     "@remix-run/express": "^2.11.2",
     "@remix-run/node": "^2.9.2",
     "@remix-run/react": "^2.9.2",
+    "@rowanmanning/feed-parser": "^1.1.0",
     "@uidotdev/usehooks": "^2.4.1",
     "bootstrap": "^5.3.3",
     "compression": "^1.7.4",

package.json

@@ -2,7 +2,7 @@ featured: false
 handle: ap-dates
 title: Associated Press Date Format
 tags:
-- date
+- dates
 variations:
   - title: "AP Date (standalone)"
     regex: '^(January|February|March|April|May|June|July|August|September|October|November|December) \d{1,2}, \d{4}$'

patterns/ap-dates.yaml

@@ -2,7 +2,7 @@ featured: true
 handle: iso8601-date
 title: ISO 8601 Date Format
 tags:
-- date
+- dates
 todo: "Handle millis, missing separators, etc."
 variations:
   - title: "ISO 8601 Complete"