Skip to content

Instantly share code, notes, and snippets.

Embed
What would you like to do?
WakaTime GitHub Commit Timeline
""" File: views.py
"""
@blueprint.route('/project/<path:project_name>/commits')
@api_utils.nocache
@auth.login_required
def project_commits(project_name):
user = app.current_user
limit, offset, page = api_utils.get_page(request, limit=35, include_page=True)
timezone = user.timezone
try:
timezone = pytz.timezone(user.timezone)
except:
abort(400)
project = Project.query.filter_by(user_id=user.id, name=project_name).first()
if project is None:
abort(404)
next_url = u('/project/{project_name}/commits').format(
project_name=project_name,
)
auth_token = user.auth_tokens.filter_by(provider='github').order_by(AuthToken.created_at.desc()).first()
if not auth_token or not auth_token.is_active:
url = oauth_utils.add_params_to_url('/oauth/github/authorize', params={
'reason': 'history',
'next': next_url,
})
return redirect(url)
scopes = auth_token.scopes.split(',') if auth_token.scopes else []
if 'repo' not in scopes:
url = oauth_utils.add_params_to_url('/oauth/github/authorize', params={
'reason': 'history',
'next': next_url,
})
return redirect(url)
app.logger.debug(auth_token.access_token)
status = user.profile.flag('commit-history-status:github')
if status != 'updating' and status != 'pending_update' and (not app.config['DEBUG'] or status != 'ok'):
if page == 1:
status = 'pending_update'
user.profile.flag('commit-history-status:github', status)
db.session.commit()
app.logger.debug('Updating github repos...')
tasks.fetch_github_commit_history.delay(user_id=user.id, auth_token_id=auth_token.id)
repository = Repository.query.filter_by(user_id=user.id, name=project.name, provider='github').order_by(Repository.external_modified_at.desc()).first()
commits = []
total_pages = 1
if repository:
app.logger.debug('Getting time per commit...')
commits = repository.get_commits(limit=limit, offset=offset)
query = repository.commits.order_by(Commit.author_date.desc())
(total, total_pages) = api_utils.get_total(query, limit=limit)
if len(commits) > 0:
start_timestamp = commits[-1].author_date
start_timestamp = start_timestamp.replace(tzinfo=pytz.utc)
start_timestamp = calendar.timegm(start_timestamp.utctimetuple())
end_timestamp = commits[0].author_date
end_timestamp = end_timestamp.replace(tzinfo=pytz.utc)
end_timestamp = calendar.timegm(end_timestamp.utctimetuple())
branches = []
(durations, filters) = api_utils.get_durations(
user_id=user.id,
timeout=user.timeout,
writes_only=user.writes_only,
start=start_timestamp,
end=end_timestamp,
project=project_name,
branches=branches,
)
durations = api_utils.combine_durations(durations, filters, project=project_name, branches=branches)
starting_at = start_timestamp
for commit in commits[::-1]: # loop through reversed commits
if commit.author_username and commit.author_username == auth_token.profile_username:
commit.calculate_logged_time(durations['durations'], starting_at=starting_at)
commit.is_current_author = True
starting_at = calendar.timegm(commit.author_date.utctimetuple())
else:
commit.is_current_author = False
commit.calculate_logged_time([], starting_at=starting_at)
context = {
'user': user,
'status': status,
'use_project_slug': False,
'project': project,
'branches': api_utils.get_branches_from_request(request),
'repository': repository,
'commits': commits,
'page': page,
'prev_page': (page - 1) if int(page) > 1 else None,
'next_page': (page + 1) if int(page) < total_pages else None,
'total_pages': total_pages,
}
if request.args.get('start'):
context['start'] = request.args.get('start')
if request.args.get('end'):
context['end'] = request.args.get('end')
return render_template('project/commits.html', **context)
""" File: tasks.py
"""
@celery.task(ignore_result=True)
def fetch_github_commit_history(user_id=None, auth_token_id=None):
if user_id is None:
raise Exception('Missing event user_id argument.')
user = User.query.filter_by(id=user_id).first()
if user is None:
raise Exception('User does not exist.')
provider = 'GitHub'
auth_token = None
if auth_token_id is not None:
auth_token = user.auth_tokens.filter_by(provider=provider.lower(), id=auth_token_id).first()
if auth_token is None or not auth_token.is_active:
auth_token = user.auth_tokens.filter_by(provider=provider.lower()).order_by(AuthToken.created_at.desc()).first()
if auth_token is None or not auth_token.is_active:
auth_token = None
scopes = []
if auth_token and auth_token.scopes:
scopes.extend(auth_token.scopes.split(','))
if 'repo' not in scopes:
auth_token = None
user.profile.flag('commit-history-status:{0}'.format(provider.lower()), 'updating')
db.session.commit()
if auth_token is None:
user.profile.flag('commit-history-status:{0}'.format(provider.lower()), 'error')
db.session.commit()
raise Exception('Missing valid AuthToken.')
try:
repos = get_or_create_github_repos(user, auth_token)
if len(repos) > 0:
ok = update_github_commits(user, auth_token, repos)
if ok:
user.profile.flag('commit-history-status:{0}'.format(provider.lower()), 'ok')
else:
user.profile.flag('commit-history-status:{0}'.format(provider.lower()), 'error')
else:
user.profile.flag('commit-history-status:{0}'.format(provider.lower()), 'empty')
db.session.commit()
except:
logger.info(u'Exception occured: {0}({1})'.format(sys.exc_info()[0].__name__, sys.exc_info()[1]))
try:
db.session.rollback()
db.session.rollback()
except:
pass
try:
user = User.query.filter_by(id=user_id).first() # get user object again because profile will have been removed from session
user.profile.flag('commit-history-status:{0}'.format(provider.lower()), 'error')
db.session.commit()
except:
pass
retry = sys.exc_info()[0].__name__ in app.config['CELERY_RETRY_EXCEPTIONS'] or 'SoftTimeLimitExceeded()' in str(sys.exc_info()[1])
if not retry:
raise sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2]
def get_or_create_github_repos(user, auth_token):
"""Get all repos for this user using GitHub api, supporting pagination.
"""
repos = []
provider = 'GitHub'
headers = {
'Accept': 'Accept: application/vnd.github.moondragon+json',
'Authorization': u('token {0}').format(auth_token.access_token),
}
url = '{url}/user/repos?per_page=100'.format(
url='https://api.github.com',
)
etag = user.profile.flag('commit-history-repos-etag:{0}'.format(provider.lower()))
app.logger.debug(etag)
if etag:
headers['If-None-Match'] = '"{0}"'.format(etag)
etag = None
while True:
response = requests.get(url, headers=headers)
# if repos have not changed since last time, return repos from our db
if response.status_code == 304:
return user.repositories.filter_by(provider=provider.lower()).all()
if response.status_code == 401:
auth_token.has_expired = True
db.session.commit()
return []
if response.status_code == 403:
return []
if response.status_code != 200:
oauth_utils.log_error(provider, code=response.status_code, traceback=response.text)
return []
if etag is None:
etag = response.headers['ETag'].replace('W/', '', 1).strip('"')
# delete etag header for next requests
if 'If-None-Match' in headers:
del headers['If-None-Match']
for repo in response.json():
defaults = {
'name': repo['name'],
'url': repo['url'],
'html_url': repo['html_url'],
'is_fork': repo['fork'],
'is_private': repo['private'],
'external_created_at': datetime.strptime(repo['created_at'], '%Y-%m-%dT%H:%M:%SZ'),
'external_modified_at': datetime.strptime(repo['updated_at'], '%Y-%m-%dT%H:%M:%SZ'),
}
if repo.get('description'):
defaults['description'] = repo['description']
if repo.get('homepage'):
defaults['homepage'] = repo['homepage']
if repo.get('stargazers_count'):
defaults['star_count'] = repo['stargazers_count']
if repo.get('forks_count'):
defaults['fork_count'] = repo['forks_count']
if repo.get('watchers_count'):
defaults['watch_count'] = repo['watchers_count']
if repo.get('default_branch'):
defaults['default_branch'] = repo['default_branch']
if 'language' in repo:
defaults['language'] = repo['language']
r = Repository.get_or_create(
defaults=defaults,
user_id=user.id,
provider=provider.lower(),
full_name=repo['full_name'],
)
r.set_columns(**defaults)
db.session.commit()
repos.append(r)
url = None
if response.headers.get('Link'):
link = response.headers.get('Link').split(';')[1]
if 'rel="next"' in link or 'rel="last"' in link:
url = response.headers.get('Link').split(';')[0].lstrip('<').rstrip('>')
if url is None:
break
if etag:
user.profile.flag('commit-history-repos-etag:{0}'.format(provider.lower()), etag)
db.session.commit()
return repos
def update_github_commits(user, auth_token, repos):
"""Create commits for this user's repos using GitHub api, supporting
pagination. Return True if all commits were created, otherwise return
False.
"""
provider = 'GitHub'
headers = {
'Accept': 'Accept: application/vnd.github.v3+json',
'Authorization': u('token {0}').format(auth_token.access_token),
}
for repo in repos:
url = '{url}/commits?per_page=100'.format(
url=repo.url,
)
etag = repo.commits_etag
if etag:
headers['If-None-Match'] = '"{0}"'.format(etag)
etag = None
page = 1
while True:
response = requests.get(url, headers=headers)
# if commits have not changed since last time, skip these commits
if response.status_code == 304:
break
# repo has not been setup
if response.status_code == 409:
break
if response.status_code == 403:
break
if response.status_code == 401:
auth_token.has_expired = True
db.session.commit()
return False
if response.status_code == '404':
repo.commits.delete()
db.session.delete(repo)
db.session.commit()
break
if response.status_code != 200:
oauth_utils.log_error(provider, code=response.status_code, traceback=response.text)
return False
if etag is None:
etag = response.headers['ETag'].replace('W/', '', 1).strip('"')
# delete etag header for next requests
if 'If-None-Match' in headers:
del headers['If-None-Match']
for commit in response.json():
defaults = {
'author_date': datetime.strptime(commit['commit']['author']['date'], '%Y-%m-%dT%H:%M:%SZ'),
'committer_date': datetime.strptime(commit['commit']['committer']['date'], '%Y-%m-%dT%H:%M:%SZ'),
'url': commit['url'],
'html_url': commit['html_url'],
'message': commit['commit']['message'],
}
if commit.get('author'):
if commit.get('author', {}).get('avatar_url'):
defaults['author_avatar_url'] = commit['author']['avatar_url']
if commit.get('author', {}).get('url'):
defaults['author_url'] = commit['author']['url']
if commit.get('author', {}).get('html_url'):
defaults['author_html_url'] = commit['author']['html_url']
if commit.get('author', {}).get('login'):
defaults['author_username'] = commit['author']['login']
if commit['commit'].get('author', {}).get('name'):
defaults['author_name'] = commit['commit']['author']['name']
if commit['commit'].get('author', {}).get('email'):
defaults['author_email'] = commit['commit']['author']['email']
if commit.get('committer'):
if commit.get('committer', {}).get('avatar_url'):
defaults['committer_avatar_url'] = commit['committer']['avatar_url']
if commit.get('committer', {}).get('url'):
defaults['committer_url'] = commit['committer']['url']
if commit.get('committer', {}).get('html_url'):
defaults['committer_html_url'] = commit['committer']['html_url']
if commit.get('committer', {}).get('login'):
defaults['committer_username'] = commit['committer']['login']
if commit['commit'].get('committer', {}).get('name'):
defaults['committer_name'] = commit['commit']['committer']['name']
if commit['commit'].get('committer', {}).get('email'):
defaults['committer_email'] = commit['commit']['committer']['email']
c = Commit.get_or_create(
defaults=defaults,
user_id=user.id,
hash=commit['sha'],
repository_id=repo.id,
)
c.set_columns(**defaults)
db.session.commit()
url = None
if response.headers.get('Link'):
link = response.headers.get('Link').split(';')[1]
if 'rel="next"' in link or 'rel="last"' in link:
url = response.headers.get('Link').split(';')[0].lstrip('<').rstrip('>')
if url is None:
break
page += 1
if page > 100:
break
if etag:
repo.commits_etag = etag
db.session.commit()
return True
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
You can’t perform that action at this time.