Source code for changes.jobs.sync_repo
from __future__ import absolute_import, print_function
import logging
from datetime import datetime
from changes.config import db
from changes.jobs.signals import fire_signal
from changes.models.repository import Repository, RepositoryBackend, RepositoryStatus
from changes.models.revision import Revision
from changes.queue.task import tracked_task
from changes.vcs.base import ConcurrentUpdateError
logger = logging.getLogger('repo.sync')
NUM_RECENT_COMMITS = 30
@tracked_task(max_retries=None)
[docs]def sync_repo(repo_id, continuous=True):
repo = Repository.query.get(repo_id)
if not repo:
logger.error('Repository %s not found', repo_id)
return False
if sync(repo) and continuous:
raise sync_repo.NotFinished(retry_after=20)
def sync(repo):
"""
Checks the repository for new commits, and fires revision.created signals.
"""
vcs = repo.get_vcs()
if vcs is None:
logger.warning('Repository %s has no VCS backend set', repo.id)
return False
if repo.status != RepositoryStatus.active:
logger.info('Repository %s is not active', repo.id)
return False
Repository.query.filter(
Repository.id == repo.id,
).update({
'last_update_attempt': datetime.utcnow(),
}, synchronize_session=False)
db.session.commit()
if vcs.exists():
try:
vcs.update()
except ConcurrentUpdateError:
# Updating already so no need to update.
pass
else:
vcs.clone()
# The loop below do two things:
# 1) adds new revisions to the database
# 2) fire off revision created signals for recent revisions
#
# TODO(dcramer): this doesnt scrape everything, and really we wouldn't
# want to do this all in a single job so we should split this into a
# backfill task
if repo.backend == RepositoryBackend.git:
revisions = vcs.log(parent=None, limit=NUM_RECENT_COMMITS, first_parent=False)
else:
revisions = vcs.log(parent=None, limit=NUM_RECENT_COMMITS)
for commit in revisions:
known_revision = Revision.query.filter(
Revision.repository_id == repo.id,
Revision.sha == commit.id
).with_for_update().scalar()
if known_revision and known_revision.date_created_signal:
db.session.commit()
continue
revision, created, _ = commit.save(repo)
db.session.commit()
# Lock the revision.
revision = Revision.query.filter(
Revision.repository_id == repo.id,
Revision.sha == commit.id
).with_for_update().scalar()
# Fire the signal if the revision was created or its branches were discovered.
#
# The `revision.branches` check is a hack right now to prevent builds from
# triggering on branchless commits.
if revision.branches and not revision.date_created_signal:
revision.date_created_signal = datetime.utcnow()
fire_signal.delay(
signal='revision.created',
kwargs={'repository_id': repo.id.hex,
'revision_sha': revision.sha},
)
db.session.commit()
db.session.commit()
Repository.query.filter(
Repository.id == repo.id,
).update({
'last_update': datetime.utcnow(),
}, synchronize_session=False)
db.session.commit()
return True