feat: daily paper rss generator with dockerfile + entrypoint
This commit is contained in:
Kunthawat Greethong
38
.gitignore
vendored
38
.gitignore
vendored
@@ -1,37 +1,9 @@
|
||||
# Byte-compiled / optimized files
|
||||
# Python
|
||||
__pycache__/
|
||||
*.py[cod]
|
||||
*$py.class
|
||||
|
||||
# Distribution / packaging
|
||||
*.egg-info/
|
||||
dist/
|
||||
build/
|
||||
|
||||
# Virtual environments
|
||||
venv/
|
||||
env/
|
||||
.env/
|
||||
.venv/
|
||||
venv/
|
||||
|
||||
# IDE specific files
|
||||
.idea/
|
||||
.vscode/
|
||||
*.swp
|
||||
*.swo
|
||||
|
||||
# OS specific files
|
||||
.DS_Store
|
||||
Thumbs.db
|
||||
|
||||
# Local testing/development
|
||||
*.bak
|
||||
*.tmp
|
||||
.coverage
|
||||
htmlcov/
|
||||
|
||||
# Exclude the HTML file downloaded from HF (if you keep test files locally)
|
||||
*Daily Papers - Hugging Face.html
|
||||
|
||||
# Uncomment the line below if you don't want to commit generated feed files
|
||||
# *.xml
|
||||
# Local test output
|
||||
feeds/
|
||||
test_*/
|
||||
|
||||
47
Dockerfile
Normal file
47
Dockerfile
Normal file
@@ -0,0 +1,47 @@
|
||||
# ─────────────────────────────────────────────
|
||||
# HuggingFace Daily Paper RSS — Dockerfile
|
||||
# ─────────────────────────────────────────────
|
||||
# Two modes:
|
||||
# 1. Run-once (default, for cron):
|
||||
# docker run --rm -v ./feeds:/data hf-paper-rss
|
||||
# 2. HTTP server (serve feeds):
|
||||
# docker run --rm -p 8080:8080 -v ./feeds:/data hf-paper-rss serve
|
||||
# ─────────────────────────────────────────────
|
||||
|
||||
FROM python:3.11-slim AS base
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
# Install system deps
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
ca-certificates \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Install Python deps
|
||||
COPY requirements.txt .
|
||||
RUN pip install --no-cache-dir -r requirements.txt
|
||||
|
||||
# Copy app code
|
||||
COPY parser.py rss_generator.py run.py run.sh ./
|
||||
|
||||
# Create output directory
|
||||
RUN mkdir -p /data && chmod 777 /data
|
||||
|
||||
# Health check
|
||||
HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \
|
||||
CMD python -c "import sys; sys.exit(0 if __import__('os').path.exists('/data/feed.xml') else 1)" \
|
||||
|| true # Allow healthcheck to be optional
|
||||
|
||||
ENTRYPOINT ["python", "run.py"]
|
||||
CMD ["--output-dir", "/data"]
|
||||
|
||||
|
||||
# ─── Serve mode ───
|
||||
FROM base AS serve
|
||||
|
||||
COPY server.py .
|
||||
|
||||
EXPOSE 8080
|
||||
|
||||
ENTRYPOINT ["python", "server.py"]
|
||||
CMD ["--dir", "/data", "--port", "8080"]
|
||||
92
run.py
Normal file
92
run.py
Normal file
@@ -0,0 +1,92 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
run.py — entrypoint for HF Daily Paper RSS generation.
|
||||
|
||||
Generates:
|
||||
- Daily feed (always)
|
||||
- Weekly feed (every Monday)
|
||||
- Monthly feed (1st of month)
|
||||
|
||||
Usage:
|
||||
python run.py # output to ./feeds/
|
||||
python run.py --output-dir /data/feeds
|
||||
python run.py --date 2026-06-27 # simulate date
|
||||
"""
|
||||
import argparse
|
||||
import datetime
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="HF Daily Paper RSS Generator")
|
||||
parser.add_argument("--output-dir", default=os.path.join(os.getcwd(), "feeds"),
|
||||
help="Directory to save RSS feeds")
|
||||
parser.add_argument("--date", default=None,
|
||||
help="Simulate a specific date (YYYY-MM-DD)")
|
||||
args = parser.parse_args()
|
||||
|
||||
output_dir = os.path.abspath(args.output_dir)
|
||||
os.makedirs(output_dir, exist_ok=True)
|
||||
|
||||
# Resolve date
|
||||
if args.date:
|
||||
today = datetime.date.fromisoformat(args.date)
|
||||
else:
|
||||
today = datetime.date.today()
|
||||
|
||||
year = today.year
|
||||
month = f"{today.month:02d}"
|
||||
week_num = today.isocalendar()[1]
|
||||
is_monday = today.isoweekday() == 1
|
||||
is_first = today.day == 1
|
||||
|
||||
script_dir = os.path.dirname(os.path.abspath(__file__))
|
||||
parser_py = os.path.join(script_dir, "parser.py")
|
||||
|
||||
print(f"[run.py] Date: {today.isoformat()} "
|
||||
f"(Year={year} Month={month} Week={week_num} "
|
||||
f"Mon={is_monday} First={is_first})")
|
||||
print(f"[run.py] Output: {output_dir}")
|
||||
|
||||
def run_parser(source, outfile):
|
||||
full_path = os.path.join(output_dir, outfile)
|
||||
print(f"\n── Generating {outfile} ──")
|
||||
result = subprocess.run(
|
||||
[sys.executable, parser_py, "--source", source, "--output", full_path],
|
||||
capture_output=False, text=True, cwd=script_dir,
|
||||
)
|
||||
return result.returncode
|
||||
|
||||
# 1. Daily (always)
|
||||
run_parser("https://huggingface.co/papers", "feed.xml")
|
||||
|
||||
# 2. Weekly (Mondays)
|
||||
if is_monday:
|
||||
run_parser(
|
||||
f"https://huggingface.co/papers/week/{year}-W{week_num}",
|
||||
"feed_weekly.xml",
|
||||
)
|
||||
else:
|
||||
print("[run.py] Skipping weekly (not Monday)")
|
||||
|
||||
# 3. Monthly (1st)
|
||||
if is_first:
|
||||
run_parser(
|
||||
f"https://huggingface.co/papers/month/{year}-{month}",
|
||||
"feed_monthly.xml",
|
||||
)
|
||||
else:
|
||||
print("[run.py] Skipping monthly (not 1st)")
|
||||
|
||||
print(f"\n[run.py] ✅ Done — feeds in {output_dir}")
|
||||
for f in sorted(os.listdir(output_dir)):
|
||||
if f.endswith(".xml"):
|
||||
fpath = os.path.join(output_dir, f)
|
||||
size = os.path.getsize(fpath)
|
||||
print(f" {f} ({size:,} bytes)")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
16
run.sh
Normal file
16
run.sh
Normal file
@@ -0,0 +1,16 @@
|
||||
#!/bin/bash
|
||||
set -euo pipefail
|
||||
# ─────────────────────────────────────────────
|
||||
# HuggingFace Daily Paper RSS — Entrypoint Script
|
||||
# ─────────────────────────────────────────────
|
||||
# Convenience wrapper — delegates to run.py
|
||||
#
|
||||
# Usage:
|
||||
# ./run.sh # output to ./feeds/
|
||||
# ./run.sh --output-dir /data/feeds
|
||||
# ./run.sh --date 2026-06-27 # simulate date
|
||||
# ─────────────────────────────────────────────
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
PYTHON=$(command -v python3 || command -v python)
|
||||
exec "$PYTHON" "$SCRIPT_DIR/run.py" "$@"
|
||||
82
server.py
Normal file
82
server.py
Normal file
@@ -0,0 +1,82 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Minimal HTTP server to serve generated RSS feed files.
|
||||
|
||||
Usage:
|
||||
python server.py # port 8080, dir ./feeds
|
||||
python server.py --dir /data --port 8080
|
||||
"""
|
||||
import argparse
|
||||
import http.server
|
||||
import os
|
||||
import sys
|
||||
import datetime
|
||||
|
||||
|
||||
class RSSFeedHandler(http.server.SimpleHTTPRequestHandler):
|
||||
"""Custom handler with CORS headers and auto-refresh."""
|
||||
|
||||
def __init__(self, *args, directory=None, **kwargs):
|
||||
super().__init__(*args, directory=directory, **kwargs)
|
||||
|
||||
def end_headers(self):
|
||||
# CORS — allow RSS readers to fetch from any origin
|
||||
self.send_header("Access-Control-Allow-Origin", "*")
|
||||
self.send_header("Access-Control-Allow-Methods", "GET, OPTIONS")
|
||||
self.send_header("Cache-Control", "no-cache, no-store, must-revalidate")
|
||||
self.send_header("Pragma", "no-cache")
|
||||
self.send_header("Expires", "0")
|
||||
super().end_headers()
|
||||
|
||||
def do_OPTIONS(self):
|
||||
self.send_response(204)
|
||||
self.end_headers()
|
||||
|
||||
def log_message(self, format, *args):
|
||||
timestamp = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
|
||||
sys.stderr.write(f"[{timestamp}] {args[0]} {args[1]} {args[2]}\n")
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="RSS Feed HTTP Server")
|
||||
parser.add_argument("--dir", default=os.path.join(os.getcwd(), "feeds"),
|
||||
help="Directory containing RSS feeds")
|
||||
parser.add_argument("--port", type=int, default=8080,
|
||||
help="Port to serve on")
|
||||
parser.add_argument("--host", default="0.0.0.0",
|
||||
help="Host to bind to")
|
||||
args = parser.parse_args()
|
||||
|
||||
feed_dir = os.path.abspath(args.dir)
|
||||
if not os.path.exists(feed_dir):
|
||||
print(f"[server.py] Creating feed directory: {feed_dir}")
|
||||
os.makedirs(feed_dir, exist_ok=True)
|
||||
|
||||
# Ensure at least one feed exists
|
||||
feeds = sorted(f for f in os.listdir(feed_dir) if f.endswith(".xml"))
|
||||
if feeds:
|
||||
print(f"[server.py] Found {len(feeds)} feed(s): {', '.join(feeds)}")
|
||||
else:
|
||||
print(f"[server.py] No feeds yet in {feed_dir}. Run 'run.py' first or wait for cron.")
|
||||
|
||||
class Handler(RSSFeedHandler):
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, directory=feed_dir, **kwargs)
|
||||
|
||||
server = http.server.HTTPServer((args.host, args.port), Handler)
|
||||
print(f"[server.py] Serving RSS feeds from {feed_dir}")
|
||||
print(f"[server.py] Listening on http://{args.host}:{args.port}")
|
||||
print(f"[server.py] Feed URLs:")
|
||||
print(f" http://localhost:{args.port}/feed.xml (daily)")
|
||||
print(f" http://localhost:{args.port}/feed_weekly.xml (weekly)")
|
||||
print(f" http://localhost:{args.port}/feed_monthly.xml (monthly)")
|
||||
|
||||
try:
|
||||
server.serve_forever()
|
||||
except KeyboardInterrupt:
|
||||
print("\n[server.py] Shutting down...")
|
||||
server.server_close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user