Compare commits
2 Commits
47130f30d7
...
b8b8e42b8e
Author | SHA1 | Date | |
---|---|---|---|
|
b8b8e42b8e | ||
|
52238ad9ae |
@ -1,4 +1,3 @@
|
||||
from itertools import chain
|
||||
|
||||
from dict_dl import Queue, WordParser, ot, rb, uq, uqall, only_first_text
|
||||
|
||||
|
3
d.py
3
d.py
@ -1,10 +1,7 @@
|
||||
#!/bin/python
|
||||
import os
|
||||
import sys
|
||||
from itertools import zip_longest
|
||||
|
||||
from rich.console import Console
|
||||
from rich.table import Table
|
||||
|
||||
from dict_dl import DictFile
|
||||
|
||||
|
3
main.py
3
main.py
@ -1,8 +1,7 @@
|
||||
import random
|
||||
from pathlib import Path
|
||||
|
||||
import uvicorn
|
||||
from fastapi import FastAPI, Response
|
||||
from fastapi import FastAPI
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
from sqlmodel import Session, select, text
|
||||
|
||||
|
@ -1,5 +1,4 @@
|
||||
from dict_dl import Queue, WordParser, cw, ot, rb, uq, uqall
|
||||
import itertools
|
||||
|
||||
|
||||
class MerriamWebsterParser(WordParser):
|
||||
|
36
mw_browser.py
Normal file
36
mw_browser.py
Normal file
@ -0,0 +1,36 @@
|
||||
import random
|
||||
from pathlib import Path
|
||||
from time import sleep
|
||||
from string import ascii_lowercase
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from requests_html import HTMLSession
|
||||
|
||||
letters = list(ascii_lowercase) + ["BIO", "GEO", "0-9"]
|
||||
db_file = "ALL.db"
|
||||
if not Path(db_file).is_file():
|
||||
Path(db_file).touch()
|
||||
|
||||
html_session = HTMLSession()
|
||||
url_stem = "https://www.merriam-webster.com"
|
||||
|
||||
links = set()
|
||||
words = {line.strip() for line in open(db_file, "rt")}
|
||||
|
||||
for letter in letters:
|
||||
url = f"https://www.merriam-webster.com/browse/dictionary/{letter}"
|
||||
r = html_session.get(url)
|
||||
# r.html.render()
|
||||
links |= {
|
||||
p.attrs["href"] for p in r.html.find(f'a[href*="browse/dictionary/{letter}/"]')
|
||||
}
|
||||
for link in links:
|
||||
print(link)
|
||||
r = html_session.get(url_stem + link)
|
||||
words |= {
|
||||
Path(urlparse(word.attrs["href"]).path).name
|
||||
for word in r.html.find('a[href*="/dictionary/"]')
|
||||
}
|
||||
sleep(random.random() * 2)
|
||||
with open(db_file, "wt") as f:
|
||||
f.write("\n".join(list(words)))
|
@ -6,7 +6,7 @@ from time import sleep
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from requests_html import HTMLSession
|
||||
from sqlmodel import Field, Relationship, Session, SQLModel, create_engine, select, Column, ARRAY, String
|
||||
from sqlmodel import Field, Session, SQLModel, create_engine, select, Column, ARRAY, String
|
||||
|
||||
with open("db.secrets", "r") as f:
|
||||
db_pass = f.readline().strip()
|
||||
|
Loading…
Reference in New Issue
Block a user