Skip to content
Open
Show file tree
Hide file tree
Changes from 3 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions tools/hlt_client/hlt_client/client.py
Original file line number Diff line number Diff line change
Expand Up @@ -184,6 +184,9 @@ def _parse_arguments():
help='Number of replays to fetch')
replay_user_parser.add_argument('-d', '--destination', dest='destination', action='store', type=str, required=True,
help="In which folder to store all resulting replay files.")
replay_user_parser.add_argument('-t', '--date', action='store', type=str, dest='date', required=True,
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This should not be a required argument.

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Okay, I will change it to 'default=None' and remove 'required=True'.

help="Fetch replay files matching the specified date. To fetch a day's files user"
"the YYYYMMDD format.")
# .Modes.Replay.Modes.Date
replay_regex_parser = replay_subparser.add_parser(REPLAY_MODE_DATE, help='Retrieve replays based on regex')
replay_regex_parser.add_argument('-t', '--date', action='store', type=str, dest='date', required=True,
Expand Down
27 changes: 20 additions & 7 deletions tools/hlt_client/hlt_client/download_game.py
Original file line number Diff line number Diff line change
Expand Up @@ -128,30 +128,41 @@ class UserGameDownloader(GameDownloader):
_FETCH_THRESHOLD = 250
_BUCKETS = []

def __init__(self, destination, user_id, limit):
def __init__(self, destination, user_id, limit, date):
"""
Download games for a user
:param destination: Where to download
:param user_id: Which user's replays to fetch
:param limit: How many replays to fetch (max)
:param date: Which date to download
"""
self.destination = destination
self.objects = self._parse_user_metadata(self._fetch_metadata(user_id, limit))
self.objects = self._parse_user_metadata(self._fetch_metadata(user_id, limit, date))

def _fetch_metadata(self, user_id, limit):
def _fetch_metadata(self, user_id, limit, date):
"""
Retrieves paginated game metadata from the halite servers for a specified user up to limit items
:param user_id: The id of the user to fetch
:param limit: The maximum number of items to fetch
:param date: Which date to download
:return: The full metadata of items
"""
print('Fetching Metadata')
current = 0
result_set = []
while current <= limit:
current_limit = self._FETCH_THRESHOLD if ((limit - current) >= self._FETCH_THRESHOLD) else (limit - current)
result_set += requests.get(self._USER_BOT_URI.format(user_id, current_limit, current)).json()
current += self._FETCH_THRESHOLD
if date is None:
current_limit = self._FETCH_THRESHOLD if ((limit - current) >= self._FETCH_THRESHOLD) else (limit - current)
result_set += requests.get(self._USER_BOT_URI.format(user_id, current_limit, current)).json()
current += self._FETCH_THRESHOLD
else:
if requests.get(self._USER_BOT_URI.format(user_id, 1, current)).json()[0]["replay"].split("-")[1] != date:
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Magic number

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I was trying to check the date of the retrieved replay one at a time. Do you have any suggestion on how to do this faster? I can also retrieve current_limit replays at a time and filter out replays that belongs to the wrong date and append the filtered list to the list result_set.

current += 1
limit += 1
continue
else:
result_set += requests.get(self._USER_BOT_URI.format(user_id, 1, current)).json()
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Magic number

current += 1
print('Finished metadata fetch. Found {} game files.'.format(len(result_set)))
return result_set

Expand Down Expand Up @@ -197,5 +208,7 @@ def download(mode, destination, date, all_bots, default_user_id, user_id, limit)
elif mode == client.REPLAY_MODE_USER:
if not (default_user_id or user_id):
raise ValueError("Cannot run default mode without authenticating .Please run `client.py --auth` first.")
UserGameDownloader(destination, default_user_id if not user_id else user_id, limit).get_objects()
if date != None and not _valid_date(date):
raise ValueError("Date must match format YYYYMMDD")
UserGameDownloader(destination, default_user_id if not user_id else user_id, limit, date).get_objects()
print('Finished writing files to desired location')