mirror of
https://github.com/andwati/pwnforfunandprofit.xyz.git
synced 2026-10-31 16:22:15 +00:00
74 lines
2.4 KiB
Python
74 lines
2.4 KiB
Python
import os
|
|
import urllib.request
|
|
import xml.etree.ElementTree as ET
|
|
import re
|
|
import html
|
|
try:
|
|
from markdownify import markdownify as md
|
|
except ImportError:
|
|
import subprocess
|
|
import sys
|
|
subprocess.check_call([sys.executable, "-m", "pip", "install", "markdownify", "beautifulsoup4"])
|
|
from markdownify import markdownify as md
|
|
from bs4 import BeautifulSoup
|
|
from datetime import datetime
|
|
|
|
FEED_URL = "https://pwnforfunandprofit.substack.com/feed"
|
|
OUTPUT_DIR = "content/"
|
|
|
|
def fetch_feed():
|
|
print(f"Fetching {FEED_URL}...")
|
|
req = urllib.request.Request(FEED_URL, headers={'User-Agent': 'Mozilla/5.0'})
|
|
with urllib.request.urlopen(req) as response:
|
|
return response.read()
|
|
|
|
def parse_and_save(xml_content):
|
|
print("Parsing RSS feed...")
|
|
root = ET.fromstring(xml_content)
|
|
|
|
if not os.path.exists(OUTPUT_DIR):
|
|
os.makedirs(OUTPUT_DIR)
|
|
|
|
for item in root.findall(".//item"):
|
|
title = item.find("title").text
|
|
pub_date_str = item.find("pubDate").text
|
|
pub_date = datetime.strptime(pub_date_str, "%a, %d %b %Y %H:%M:%S %Z")
|
|
content_encoded = item.find("{http://purl.org/rss/1.0/modules/content/}encoded").text
|
|
|
|
# Determine slug
|
|
slug = re.sub(r'[^a-z0-9]+', '-', title.lower()).strip('-')
|
|
|
|
# HTML parsing for extra clean up
|
|
soup = BeautifulSoup(content_encoded, 'html.parser')
|
|
|
|
# Remove substack specific widgets
|
|
for widget in soup.find_all(class_='subscription-widget-wrap-editor'):
|
|
widget.decompose()
|
|
for banner in soup.find_all(class_='button-wrapper'):
|
|
banner.decompose()
|
|
|
|
clean_html = str(soup)
|
|
|
|
# Convert HTML to Markdown
|
|
markdown_content = md(clean_html, heading_style="ATX", bypass_tables=False)
|
|
|
|
# Create TOML frontmatter
|
|
frontmatter = f"""+++
|
|
title = "{title.replace('"', '\\"')}"
|
|
date = {pub_date.strftime('%Y-%m-%d')}
|
|
description = "{item.find('description').text.replace('"', '\\"') if item.find('description') is not None else ''}"
|
|
[taxonomies]
|
|
tags = ["blog"]
|
|
+++
|
|
|
|
"""
|
|
file_path = os.path.join(OUTPUT_DIR, f"{slug}.md")
|
|
with open(file_path, "w", encoding="utf-8") as f:
|
|
f.write(frontmatter + markdown_content)
|
|
print(f"Saved: {file_path}")
|
|
|
|
if __name__ == "__main__":
|
|
xml_data = fetch_feed()
|
|
parse_and_save(xml_data)
|
|
print("Migration complete!")
|