added cli arg parser; output format changed to CSV
This commit is contained in:
parent
323144200b
commit
d98e00a8ae
@ -1,7 +1,10 @@
|
||||
import logging
|
||||
import re
|
||||
import csv
|
||||
import sys
|
||||
import asyncio
|
||||
|
||||
from argparse import ArgumentParser
|
||||
from pathlib import Path
|
||||
from datetime import datetime
|
||||
from string import ascii_uppercase
|
||||
|
||||
@ -11,7 +14,7 @@ from bs4.element import Tag, ResultSet
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
log.setLevel(logging.DEBUG)
|
||||
log.setLevel(logging.ERROR)
|
||||
ch = logging.StreamHandler()
|
||||
ch.setLevel(logging.DEBUG)
|
||||
log.addHandler(ch)
|
||||
@ -105,10 +108,33 @@ async def get_all_data(asynchronous: bool = False) -> list[row_type]:
|
||||
|
||||
|
||||
def main() -> None:
|
||||
data = asyncio.run(get_all_data(True))
|
||||
for tup in data:
|
||||
print(tup)
|
||||
print(len(data), 'datasets')
|
||||
parser = ArgumentParser(description="Scrape all stock symbols")
|
||||
parser.add_argument(
|
||||
'-v', '--verbose',
|
||||
action='store_true',
|
||||
help="If set, prints all sorts of stuff."
|
||||
)
|
||||
parser.add_argument(
|
||||
'-S', '--sequential',
|
||||
action='store_true',
|
||||
help="If set, all requests are performed sequentially; otherwise async capabilities are used for concurrency."
|
||||
)
|
||||
parser.add_argument(
|
||||
'-f', '--to-file',
|
||||
type=Path,
|
||||
help="Writes results to the specified destination file. If omitted results are printed to stdout."
|
||||
)
|
||||
args = parser.parse_args()
|
||||
if args.verbose:
|
||||
log.setLevel(logging.DEBUG)
|
||||
|
||||
data = asyncio.run(get_all_data(not args.sequential))
|
||||
|
||||
if args.to_file is None:
|
||||
csv.writer(sys.stdout).writerows(data)
|
||||
else:
|
||||
with open(args.to_file, 'w') as f:
|
||||
csv.writer(f).writerows(data)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
Loading…
Reference in New Issue
Block a user