added cli arg parser; output format changed to CSV

This commit is contained in:
Daniil Fajnberg 2021-11-11 17:04:26 +01:00
parent 323144200b
commit d98e00a8ae

View File

@ -1,7 +1,10 @@
import logging
import re
import csv
import sys
import asyncio
from argparse import ArgumentParser
from pathlib import Path
from datetime import datetime
from string import ascii_uppercase
@ -11,7 +14,7 @@ from bs4.element import Tag, ResultSet
log = logging.getLogger(__name__)
log.setLevel(logging.DEBUG)
log.setLevel(logging.ERROR)
ch = logging.StreamHandler()
ch.setLevel(logging.DEBUG)
log.addHandler(ch)
@ -105,10 +108,33 @@ async def get_all_data(asynchronous: bool = False) -> list[row_type]:
def main() -> None:
data = asyncio.run(get_all_data(True))
for tup in data:
print(tup)
print(len(data), 'datasets')
parser = ArgumentParser(description="Scrape all stock symbols")
parser.add_argument(
'-v', '--verbose',
action='store_true',
help="If set, prints all sorts of stuff."
)
parser.add_argument(
'-S', '--sequential',
action='store_true',
help="If set, all requests are performed sequentially; otherwise async capabilities are used for concurrency."
)
parser.add_argument(
'-f', '--to-file',
type=Path,
help="Writes results to the specified destination file. If omitted results are printed to stdout."
)
args = parser.parse_args()
if args.verbose:
log.setLevel(logging.DEBUG)
data = asyncio.run(get_all_data(not args.sequential))
if args.to_file is None:
csv.writer(sys.stdout).writerows(data)
else:
with open(args.to_file, 'w') as f:
csv.writer(f).writerows(data)
if __name__ == '__main__':