Initial revision
This commit is contained in:
parent
ed69701311
commit
8414256088
28 changed files with 855 additions and 0 deletions
3
.gitignore
vendored
Normal file
3
.gitignore
vendored
Normal file
|
@ -0,0 +1,3 @@
|
||||||
|
__pycache__
|
||||||
|
/db.sqlite3
|
||||||
|
/giteamigrate/settings.py
|
0
giteamigrate/__init__.py
Normal file
0
giteamigrate/__init__.py
Normal file
16
giteamigrate/asgi.py
Normal file
16
giteamigrate/asgi.py
Normal file
|
@ -0,0 +1,16 @@
|
||||||
|
"""
|
||||||
|
ASGI config for giteamigrate project.
|
||||||
|
|
||||||
|
It exposes the ASGI callable as a module-level variable named ``application``.
|
||||||
|
|
||||||
|
For more information on this file, see
|
||||||
|
https://docs.djangoproject.com/en/3.0/howto/deployment/asgi/
|
||||||
|
"""
|
||||||
|
|
||||||
|
import os
|
||||||
|
|
||||||
|
from django.core.asgi import get_asgi_application
|
||||||
|
|
||||||
|
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'giteamigrate.settings')
|
||||||
|
|
||||||
|
application = get_asgi_application()
|
115
giteamigrate/settings.dist.py
Normal file
115
giteamigrate/settings.dist.py
Normal file
|
@ -0,0 +1,115 @@
|
||||||
|
import os
|
||||||
|
|
||||||
|
# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
|
||||||
|
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||||
|
|
||||||
|
GITLAB_API = 'https://git.zom.bi/api/v4'
|
||||||
|
GITEA_API = 'https://gitea.zom.bi/api/v1'
|
||||||
|
GITLAB_REPO_URL = 'https://%s:%s@git.zom.bi/%s.git'
|
||||||
|
GITEA_REPO_URL = 'https://%s:%s@gitea.zom.bi/%s.git'
|
||||||
|
|
||||||
|
# SECURITY WARNING: keep the secret key used in production secret!
|
||||||
|
SECRET_KEY = '' # Fill with key
|
||||||
|
|
||||||
|
DEBUG = False
|
||||||
|
|
||||||
|
ALLOWED_HOSTS = [
|
||||||
|
'*'
|
||||||
|
]
|
||||||
|
|
||||||
|
# Application definition
|
||||||
|
|
||||||
|
INSTALLED_APPS = [
|
||||||
|
'django.contrib.admin',
|
||||||
|
'django.contrib.auth',
|
||||||
|
'django.contrib.contenttypes',
|
||||||
|
'django.contrib.sessions',
|
||||||
|
'django.contrib.messages',
|
||||||
|
'django.contrib.staticfiles',
|
||||||
|
'migrator',
|
||||||
|
'workers',
|
||||||
|
]
|
||||||
|
|
||||||
|
MIDDLEWARE = [
|
||||||
|
'django.middleware.security.SecurityMiddleware',
|
||||||
|
'django.contrib.sessions.middleware.SessionMiddleware',
|
||||||
|
'django.middleware.common.CommonMiddleware',
|
||||||
|
'django.middleware.csrf.CsrfViewMiddleware',
|
||||||
|
'django.contrib.auth.middleware.AuthenticationMiddleware',
|
||||||
|
'django.contrib.messages.middleware.MessageMiddleware',
|
||||||
|
'django.middleware.clickjacking.XFrameOptionsMiddleware',
|
||||||
|
]
|
||||||
|
|
||||||
|
ROOT_URLCONF = 'giteamigrate.urls'
|
||||||
|
|
||||||
|
TEMPLATES = [
|
||||||
|
{
|
||||||
|
'BACKEND': 'django.template.backends.django.DjangoTemplates',
|
||||||
|
'DIRS': [],
|
||||||
|
'APP_DIRS': True,
|
||||||
|
'OPTIONS': {
|
||||||
|
'context_processors': [
|
||||||
|
'django.template.context_processors.debug',
|
||||||
|
'django.template.context_processors.request',
|
||||||
|
'django.contrib.auth.context_processors.auth',
|
||||||
|
'django.contrib.messages.context_processors.messages',
|
||||||
|
],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
]
|
||||||
|
|
||||||
|
WSGI_APPLICATION = 'giteamigrate.wsgi.application'
|
||||||
|
|
||||||
|
|
||||||
|
# Database
|
||||||
|
# https://docs.djangoproject.com/en/3.0/ref/settings/#databases
|
||||||
|
|
||||||
|
DATABASES = {
|
||||||
|
'default': {
|
||||||
|
'ENGINE': 'django.db.backends.sqlite3',
|
||||||
|
'NAME': os.path.join(BASE_DIR, 'db.sqlite3'),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
# Password validation
|
||||||
|
# https://docs.djangoproject.com/en/3.0/ref/settings/#auth-password-validators
|
||||||
|
|
||||||
|
AUTH_PASSWORD_VALIDATORS = [
|
||||||
|
{
|
||||||
|
'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator',
|
||||||
|
},
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
# Internationalization
|
||||||
|
# https://docs.djangoproject.com/en/3.0/topics/i18n/
|
||||||
|
|
||||||
|
LANGUAGE_CODE = 'en-us'
|
||||||
|
|
||||||
|
TIME_ZONE = 'UTC'
|
||||||
|
|
||||||
|
USE_I18N = True
|
||||||
|
|
||||||
|
USE_L10N = True
|
||||||
|
|
||||||
|
USE_TZ = True
|
||||||
|
|
||||||
|
|
||||||
|
# Static files (CSS, JavaScript, Images)
|
||||||
|
# https://docs.djangoproject.com/en/3.0/howto/static-files/
|
||||||
|
|
||||||
|
STATIC_URL = '/static/'
|
||||||
|
|
||||||
|
STATICFILES_DIRS = [
|
||||||
|
os.path.join(BASE_DIR, "static")
|
||||||
|
]
|
22
giteamigrate/urls.py
Normal file
22
giteamigrate/urls.py
Normal file
|
@ -0,0 +1,22 @@
|
||||||
|
"""giteamigrate URL Configuration
|
||||||
|
|
||||||
|
The `urlpatterns` list routes URLs to views. For more information please see:
|
||||||
|
https://docs.djangoproject.com/en/3.0/topics/http/urls/
|
||||||
|
Examples:
|
||||||
|
Function views
|
||||||
|
1. Add an import: from my_app import views
|
||||||
|
2. Add a URL to urlpatterns: path('', views.home, name='home')
|
||||||
|
Class-based views
|
||||||
|
1. Add an import: from other_app.views import Home
|
||||||
|
2. Add a URL to urlpatterns: path('', Home.as_view(), name='home')
|
||||||
|
Including another URLconf
|
||||||
|
1. Import the include() function: from django.urls import include, path
|
||||||
|
2. Add a URL to urlpatterns: path('blog/', include('blog.urls'))
|
||||||
|
"""
|
||||||
|
from django.contrib import admin
|
||||||
|
from django.urls import path, include
|
||||||
|
|
||||||
|
urlpatterns = [
|
||||||
|
path('', include('migrator.urls')),
|
||||||
|
path('admin/', admin.site.urls),
|
||||||
|
]
|
16
giteamigrate/wsgi.py
Normal file
16
giteamigrate/wsgi.py
Normal file
|
@ -0,0 +1,16 @@
|
||||||
|
"""
|
||||||
|
WSGI config for giteamigrate project.
|
||||||
|
|
||||||
|
It exposes the WSGI callable as a module-level variable named ``application``.
|
||||||
|
|
||||||
|
For more information on this file, see
|
||||||
|
https://docs.djangoproject.com/en/3.0/howto/deployment/wsgi/
|
||||||
|
"""
|
||||||
|
|
||||||
|
import os
|
||||||
|
|
||||||
|
from django.core.wsgi import get_wsgi_application
|
||||||
|
|
||||||
|
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'giteamigrate.settings')
|
||||||
|
|
||||||
|
application = get_wsgi_application()
|
21
manage.py
Executable file
21
manage.py
Executable file
|
@ -0,0 +1,21 @@
|
||||||
|
#!/usr/bin/env python
|
||||||
|
"""Django's command-line utility for administrative tasks."""
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'giteamigrate.settings')
|
||||||
|
try:
|
||||||
|
from django.core.management import execute_from_command_line
|
||||||
|
except ImportError as exc:
|
||||||
|
raise ImportError(
|
||||||
|
"Couldn't import Django. Are you sure it's installed and "
|
||||||
|
"available on your PYTHONPATH environment variable? Did you "
|
||||||
|
"forget to activate a virtual environment?"
|
||||||
|
) from exc
|
||||||
|
execute_from_command_line(sys.argv)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
main()
|
0
migrator/__init__.py
Normal file
0
migrator/__init__.py
Normal file
3
migrator/admin.py
Normal file
3
migrator/admin.py
Normal file
|
@ -0,0 +1,3 @@
|
||||||
|
from django.contrib import admin
|
||||||
|
|
||||||
|
# Register your models here.
|
33
migrator/api.py
Normal file
33
migrator/api.py
Normal file
|
@ -0,0 +1,33 @@
|
||||||
|
import requests
|
||||||
|
from . import settings
|
||||||
|
|
||||||
|
class API:
|
||||||
|
def __init__(self, prefix, params={}):
|
||||||
|
self.prefix=prefix
|
||||||
|
self.default_params=params
|
||||||
|
|
||||||
|
def get(self, url, *args, **kwargs):
|
||||||
|
params = kwargs.get('params', {})
|
||||||
|
for k,v in self.default_params.items():
|
||||||
|
if k not in params:
|
||||||
|
params[k] = v
|
||||||
|
kwargs['params'] = params
|
||||||
|
|
||||||
|
return requests.get(self.prefix + url, *args, **kwargs)
|
||||||
|
|
||||||
|
def post(self, url, *args, **kwargs):
|
||||||
|
params = kwargs.get('params', {})
|
||||||
|
for k,v in self.default_params.items():
|
||||||
|
if k not in params:
|
||||||
|
params[k] = v
|
||||||
|
kwargs['params'] = params
|
||||||
|
|
||||||
|
return requests.post(self.prefix + url, *args, **kwargs)
|
||||||
|
|
||||||
|
class GiteaAPI(API):
|
||||||
|
def __init__(self, token):
|
||||||
|
super().__init__(settings.GITEA_API, {'token': token})
|
||||||
|
|
||||||
|
class GitlabAPI(API):
|
||||||
|
def __init__(self, token):
|
||||||
|
super().__init__(settings.GITLAB_API, {'private_token': token})
|
5
migrator/apps.py
Normal file
5
migrator/apps.py
Normal file
|
@ -0,0 +1,5 @@
|
||||||
|
from django.apps import AppConfig
|
||||||
|
|
||||||
|
|
||||||
|
class MigratorConfig(AppConfig):
|
||||||
|
name = 'migrator'
|
11
migrator/helper.py
Normal file
11
migrator/helper.py
Normal file
|
@ -0,0 +1,11 @@
|
||||||
|
import re
|
||||||
|
|
||||||
|
def get_dict_from_query(query, name):
|
||||||
|
d = {}
|
||||||
|
matcher = re.compile("^%s\[(.+)\]$" % name)
|
||||||
|
for key, value in query.items():
|
||||||
|
match = matcher.match(key)
|
||||||
|
if match is not None:
|
||||||
|
d[match.group(1)] = value
|
||||||
|
|
||||||
|
return d
|
34
migrator/migrations/0001_initial.py
Normal file
34
migrator/migrations/0001_initial.py
Normal file
|
@ -0,0 +1,34 @@
|
||||||
|
# Generated by Django 3.0.8 on 2020-08-01 15:33
|
||||||
|
|
||||||
|
from django.db import migrations, models
|
||||||
|
import django.db.models.deletion
|
||||||
|
import migrator.models
|
||||||
|
import uuid
|
||||||
|
|
||||||
|
|
||||||
|
class Migration(migrations.Migration):
|
||||||
|
|
||||||
|
initial = True
|
||||||
|
|
||||||
|
dependencies = [
|
||||||
|
]
|
||||||
|
|
||||||
|
operations = [
|
||||||
|
migrations.CreateModel(
|
||||||
|
name='Migration',
|
||||||
|
fields=[
|
||||||
|
('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
|
||||||
|
('gitlab_token', models.CharField(max_length=200)),
|
||||||
|
('gitea_token', models.CharField(max_length=200)),
|
||||||
|
],
|
||||||
|
),
|
||||||
|
migrations.CreateModel(
|
||||||
|
name='Repository',
|
||||||
|
fields=[
|
||||||
|
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
|
||||||
|
('path_with_namespace', models.CharField(max_length=200)),
|
||||||
|
('result', models.PositiveIntegerField(choices=[(migrator.models.Status['PENDING'], 1), (migrator.models.Status['SUCCESS'], 2), (migrator.models.Status['EXISTS'], 3), (migrator.models.Status['ERROR'], 4)], default=migrator.models.Status['PENDING'])),
|
||||||
|
('migration', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='repositories', to='migrator.Migration')),
|
||||||
|
],
|
||||||
|
),
|
||||||
|
]
|
19
migrator/migrations/0002_migration_username.py
Normal file
19
migrator/migrations/0002_migration_username.py
Normal file
|
@ -0,0 +1,19 @@
|
||||||
|
# Generated by Django 3.0.8 on 2020-08-01 15:54
|
||||||
|
|
||||||
|
from django.db import migrations, models
|
||||||
|
|
||||||
|
|
||||||
|
class Migration(migrations.Migration):
|
||||||
|
|
||||||
|
dependencies = [
|
||||||
|
('migrator', '0001_initial'),
|
||||||
|
]
|
||||||
|
|
||||||
|
operations = [
|
||||||
|
migrations.AddField(
|
||||||
|
model_name='migration',
|
||||||
|
name='username',
|
||||||
|
field=models.CharField(default='anonymous', max_length=200),
|
||||||
|
preserve_default=False,
|
||||||
|
),
|
||||||
|
]
|
25
migrator/migrations/0003_auto_20200803_1947.py
Normal file
25
migrator/migrations/0003_auto_20200803_1947.py
Normal file
|
@ -0,0 +1,25 @@
|
||||||
|
# Generated by Django 3.0.8 on 2020-08-03 19:47
|
||||||
|
|
||||||
|
from django.db import migrations, models
|
||||||
|
import migrator.models
|
||||||
|
|
||||||
|
|
||||||
|
class Migration(migrations.Migration):
|
||||||
|
|
||||||
|
dependencies = [
|
||||||
|
('migrator', '0002_migration_username'),
|
||||||
|
]
|
||||||
|
|
||||||
|
operations = [
|
||||||
|
migrations.AddField(
|
||||||
|
model_name='repository',
|
||||||
|
name='error_message',
|
||||||
|
field=models.TextField(default=''),
|
||||||
|
preserve_default=False,
|
||||||
|
),
|
||||||
|
migrations.AlterField(
|
||||||
|
model_name='repository',
|
||||||
|
name='result',
|
||||||
|
field=models.PositiveIntegerField(choices=[(migrator.models.Status['PENDING'], 1), (migrator.models.Status['IN_PROGRESS'], 5), (migrator.models.Status['SUCCESS'], 2), (migrator.models.Status['EXISTS'], 3), (migrator.models.Status['ERROR'], 4)], default=migrator.models.Status['PENDING']),
|
||||||
|
),
|
||||||
|
]
|
18
migrator/migrations/0004_auto_20200803_1948.py
Normal file
18
migrator/migrations/0004_auto_20200803_1948.py
Normal file
|
@ -0,0 +1,18 @@
|
||||||
|
# Generated by Django 3.0.8 on 2020-08-03 19:48
|
||||||
|
|
||||||
|
from django.db import migrations, models
|
||||||
|
|
||||||
|
|
||||||
|
class Migration(migrations.Migration):
|
||||||
|
|
||||||
|
dependencies = [
|
||||||
|
('migrator', '0003_auto_20200803_1947'),
|
||||||
|
]
|
||||||
|
|
||||||
|
operations = [
|
||||||
|
migrations.AlterField(
|
||||||
|
model_name='repository',
|
||||||
|
name='error_message',
|
||||||
|
field=models.TextField(default=''),
|
||||||
|
),
|
||||||
|
]
|
0
migrator/migrations/__init__.py
Normal file
0
migrator/migrations/__init__.py
Normal file
35
migrator/models.py
Normal file
35
migrator/models.py
Normal file
|
@ -0,0 +1,35 @@
|
||||||
|
from django.db import models
|
||||||
|
from enum import IntEnum
|
||||||
|
import uuid
|
||||||
|
|
||||||
|
class Migration(models.Model):
|
||||||
|
id = models.UUIDField(
|
||||||
|
primary_key = True,
|
||||||
|
default = uuid.uuid4,
|
||||||
|
editable = False
|
||||||
|
)
|
||||||
|
|
||||||
|
username = models.CharField(max_length=200)
|
||||||
|
gitlab_token = models.CharField(max_length=200)
|
||||||
|
gitea_token = models.CharField(max_length=200)
|
||||||
|
|
||||||
|
class Status(IntEnum):
|
||||||
|
PENDING = 1
|
||||||
|
IN_PROGRESS = 5
|
||||||
|
SUCCESS = 2
|
||||||
|
EXISTS = 3
|
||||||
|
ERROR = 4
|
||||||
|
|
||||||
|
class Repository(models.Model):
|
||||||
|
migration = models.ForeignKey(
|
||||||
|
Migration,
|
||||||
|
on_delete = models.CASCADE,
|
||||||
|
related_name = "repositories",
|
||||||
|
)
|
||||||
|
|
||||||
|
path_with_namespace = models.CharField(max_length=200)
|
||||||
|
result = models.PositiveIntegerField(
|
||||||
|
choices=[(tag, tag.value) for tag in Status],
|
||||||
|
default = Status.PENDING
|
||||||
|
)
|
||||||
|
error_message = models.TextField(default="")
|
7
migrator/settings.py
Normal file
7
migrator/settings.py
Normal file
|
@ -0,0 +1,7 @@
|
||||||
|
from django.conf import settings
|
||||||
|
|
||||||
|
GITLAB_API = getattr(settings, 'GITLAB_API', 'https://localhost/api/v4')
|
||||||
|
GITEA_API = getattr(settings, 'GITEA_API', 'https://localhost/api/v1')
|
||||||
|
|
||||||
|
GITLAB_REPO_URL = getattr(settings, 'GITLAB_REPO_URL', 'https://%s:%s@localhost/%s.git')
|
||||||
|
GITEA_REPO_URL = getattr(settings, 'GITEA_REPO_URL', 'https://%s:%s@localhost/%s.git')
|
135
migrator/tasks.py
Normal file
135
migrator/tasks.py
Normal file
|
@ -0,0 +1,135 @@
|
||||||
|
from workers import task
|
||||||
|
from .models import *
|
||||||
|
from time import sleep
|
||||||
|
from random import randint
|
||||||
|
from tempfile import TemporaryDirectory
|
||||||
|
from . import settings
|
||||||
|
from .api import *
|
||||||
|
from os import path
|
||||||
|
import re
|
||||||
|
import subprocess
|
||||||
|
import traceback
|
||||||
|
|
||||||
|
class MigrationError(Exception):
|
||||||
|
pass
|
||||||
|
|
||||||
|
def handle_migration(repository):
|
||||||
|
gitea = GiteaAPI(repository.migration.gitea_token)
|
||||||
|
gitlab = GitlabAPI(repository.migration.gitlab_token)
|
||||||
|
|
||||||
|
match = re.search('^([^/]+)/',repository.path_with_namespace)
|
||||||
|
if match is None:
|
||||||
|
raise MigrationError("Could not get organization")
|
||||||
|
else:
|
||||||
|
organization = match.group(1)
|
||||||
|
|
||||||
|
match = re.search('([^/]+)$', repository.path_with_namespace)
|
||||||
|
if match is None:
|
||||||
|
raise MigrationError("Could not get repo name")
|
||||||
|
else:
|
||||||
|
reponame = match.group(1)
|
||||||
|
|
||||||
|
# Create organization if necessary
|
||||||
|
if organization == repository.migration.username:
|
||||||
|
organization = None
|
||||||
|
|
||||||
|
if organization is None:
|
||||||
|
response = gitea.get("/orgs/%s" % organization)
|
||||||
|
|
||||||
|
if response.status_code != 200:
|
||||||
|
response = gitea.post("/orgs", json={
|
||||||
|
"full_name" : organization,
|
||||||
|
"username" : organization,
|
||||||
|
})
|
||||||
|
|
||||||
|
if response.status_code != 201:
|
||||||
|
raise MigrationError("Could not create organization: %s" % (response.text) )
|
||||||
|
|
||||||
|
owner = organization or repository.migration.username
|
||||||
|
|
||||||
|
# Create repo if necessary
|
||||||
|
if organization is None:
|
||||||
|
response = gitea.post('/user/repos', data={
|
||||||
|
'auto_init': False,
|
||||||
|
'default_branch': 'master',
|
||||||
|
'description': "Automatically created",
|
||||||
|
'name': reponame,
|
||||||
|
})
|
||||||
|
else:
|
||||||
|
response = gitea.post('/org/%s/repos' % organization, data={
|
||||||
|
'name': reponame,
|
||||||
|
'private': True,
|
||||||
|
})
|
||||||
|
|
||||||
|
if response.status_code == 409:
|
||||||
|
pass # exists
|
||||||
|
elif response.status_code != 201:
|
||||||
|
raise MigrationError("Could not create repo: %s" % (response.text))
|
||||||
|
|
||||||
|
gitlab_remote = settings.GITLAB_REPO_URL % (
|
||||||
|
repository.migration.username,
|
||||||
|
repository.migration.gitlab_token,
|
||||||
|
repository.path_with_namespace
|
||||||
|
)
|
||||||
|
|
||||||
|
gitea_remote = settings.GITEA_REPO_URL % (
|
||||||
|
repository.migration.username,
|
||||||
|
repository.migration.gitea_token,
|
||||||
|
repository.path_with_namespace
|
||||||
|
)
|
||||||
|
|
||||||
|
# Migrate
|
||||||
|
with TemporaryDirectory() as tempdir:
|
||||||
|
gitdir = path.join(tempdir,'git')
|
||||||
|
|
||||||
|
try:
|
||||||
|
subprocess.check_call([
|
||||||
|
'git','clone','--mirror',
|
||||||
|
gitlab_remote,
|
||||||
|
gitdir
|
||||||
|
])
|
||||||
|
except subprocess.CalledProcessError:
|
||||||
|
raise MigrationError("Could not clone")
|
||||||
|
|
||||||
|
try:
|
||||||
|
subprocess.check_call([
|
||||||
|
'git', '-C', gitdir,
|
||||||
|
'remote','add',
|
||||||
|
'gitea', gitea_remote
|
||||||
|
])
|
||||||
|
except subprocess.CalledProcessError:
|
||||||
|
raise MigrationError("Could not set remote")
|
||||||
|
|
||||||
|
try:
|
||||||
|
subprocess.check_call([
|
||||||
|
'git', '-C', gitdir,
|
||||||
|
'push', '--all', 'gitea'
|
||||||
|
])
|
||||||
|
except subprocess.CalledProcessError:
|
||||||
|
raise MigrationError("Could not push")
|
||||||
|
|
||||||
|
return Status.SUCCESS
|
||||||
|
|
||||||
|
@task()
|
||||||
|
def migrate_repository(repository_id):
|
||||||
|
try:
|
||||||
|
repository = Repository.objects.select_related('migration').get(pk=repository_id)
|
||||||
|
except Repository.DoesNotExist:
|
||||||
|
return
|
||||||
|
|
||||||
|
print(">>> Migrating repository %s..." % (repository.path_with_namespace), end="\n\n")
|
||||||
|
|
||||||
|
repository.result = Status.IN_PROGRESS
|
||||||
|
repository.save()
|
||||||
|
|
||||||
|
# Might take a while
|
||||||
|
try:
|
||||||
|
result = handle_migration(repository)
|
||||||
|
repository.result = result
|
||||||
|
except Exception as e:
|
||||||
|
result = Status.ERROR
|
||||||
|
repository.result = result
|
||||||
|
repository.error_message = traceback.format_exc()
|
||||||
|
finally:
|
||||||
|
repository.save()
|
||||||
|
print(end="\n\n")
|
15
migrator/templates/base.html
Normal file
15
migrator/templates/base.html
Normal file
|
@ -0,0 +1,15 @@
|
||||||
|
{% load static %}
|
||||||
|
<html>
|
||||||
|
<head>
|
||||||
|
<title>Gitlab to Gitea migrator</title>
|
||||||
|
<link rel="stylesheet" href="{% static 'bootstrap.min.css' %}">
|
||||||
|
</head>
|
||||||
|
<body>
|
||||||
|
<div class="container mt-4">
|
||||||
|
{% block content %}
|
||||||
|
{% endblock %}
|
||||||
|
</div>
|
||||||
|
<script src="{% static 'jquery-3.5.1.min.js' %}"></script>
|
||||||
|
{% block javascript %}{% endblock %}
|
||||||
|
</body>
|
||||||
|
</html>
|
122
migrator/templates/migration_create.html
Normal file
122
migrator/templates/migration_create.html
Normal file
|
@ -0,0 +1,122 @@
|
||||||
|
{% extends 'base.html' %}
|
||||||
|
|
||||||
|
{% block content %}
|
||||||
|
<form method="post">
|
||||||
|
{% csrf_token %}
|
||||||
|
<div class="form-group row">
|
||||||
|
<label for="gitea_token" class="col-form-label col-md-auto">Gitea Access Token:</label>
|
||||||
|
<div class="col">
|
||||||
|
<input name="gitea_token"
|
||||||
|
id="gitea_token"
|
||||||
|
class="form-control">
|
||||||
|
<small class="text-muted">
|
||||||
|
Can be created
|
||||||
|
<a href="https://gitea.zom.bi/user/settings/applications">here</a>
|
||||||
|
</small>
|
||||||
|
</div>
|
||||||
|
<div class="col-md-auto">
|
||||||
|
<button type="button" class="btn btn-primary" id="buttonCheckGitea">
|
||||||
|
Check Gitea Access
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="form-group row">
|
||||||
|
<label for="gitlab_token" class="col-form-label col-md-auto">Gitlab Access Token:</label>
|
||||||
|
<div class="col">
|
||||||
|
<input name="gitlab_token"
|
||||||
|
id="gitlab_token"
|
||||||
|
class="form-control">
|
||||||
|
<small class="text-muted">
|
||||||
|
Can be created
|
||||||
|
<a href="https://git.zom.bi/profile/personal_access_tokens">here</a> (with api scope)
|
||||||
|
</small>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="form-group" id="projectfetch">
|
||||||
|
<button type="button" class="btn btn-primary" id="buttonFetch">
|
||||||
|
Fetch projects from GitLab
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="form-group d-none" id="projectselect">
|
||||||
|
<h4>Select projects to migrate</h4>
|
||||||
|
<div class="d-flex flex-row">
|
||||||
|
<div class="btn-group btn-group-sm mr-auto">
|
||||||
|
<button type="button" class="btn btn-outline-secondary"
|
||||||
|
id="buttonSelectAll">
|
||||||
|
Select all
|
||||||
|
</button>
|
||||||
|
<button type="button" class="btn btn-outline-secondary"
|
||||||
|
id="buttonSelectNone">
|
||||||
|
Select none
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
<div>
|
||||||
|
<button type="submit" class="btn btn-primary">
|
||||||
|
Start migration
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<ul id="projectslist" class="list-group mt-4">
|
||||||
|
</ul>
|
||||||
|
<input type="hidden" id="usernamefield" name="username" value="">
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</form>
|
||||||
|
{% endblock %}
|
||||||
|
|
||||||
|
{% block javascript %}
|
||||||
|
<script type="text/javascript">
|
||||||
|
$(document).ready(function() {
|
||||||
|
$('#buttonFetch').click(function() {
|
||||||
|
$('#projectselect').addClass('d-none');
|
||||||
|
$('#gitlab_token').prop('readonly', true);
|
||||||
|
$.getJSON("{% url 'fetch_gitlab_projects' %}?gitlab_token="+$('#gitlab_token').val(), function(data) {
|
||||||
|
if(data.error) {
|
||||||
|
alert(data.error);
|
||||||
|
$('#gitlab_token').prop('readonly', false);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
var items = [];
|
||||||
|
data.projects.forEach(function(project) {
|
||||||
|
items.push(
|
||||||
|
"<li class='list-group-item'>" +
|
||||||
|
"<input name='projects[" + project + "]' type=checkbox>"+
|
||||||
|
"<span class='pl-2'>" + project + "</span>"+
|
||||||
|
"</li>"
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
$('#usernamefield').val(data.username);
|
||||||
|
|
||||||
|
$('#projectslist').html(items.join(""));
|
||||||
|
$('#projectselect').removeClass('d-none');
|
||||||
|
$('#projectfetch').addClass('d-none');
|
||||||
|
$('#buttonFetch').off('click');
|
||||||
|
}).fail(function() {
|
||||||
|
$('#gitlab_token').prop('readonly', false);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
$('#buttonCheckGitea').click(function() {
|
||||||
|
$.getJSON("{% url 'check_gitea_access' %}?gitea_token="+$('#gitea_token').val(), function(result) {
|
||||||
|
if(result.success) {
|
||||||
|
alert("Gitea access works");
|
||||||
|
} else {
|
||||||
|
alert("Gitea access failed");
|
||||||
|
}
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
$('#buttonSelectAll').click(function() {
|
||||||
|
$('#projectslist input[type=checkbox]').prop('checked',true);
|
||||||
|
});
|
||||||
|
|
||||||
|
$('#buttonSelectNone').click(function() {
|
||||||
|
$('#projectslist input[type=checkbox]').prop('checked',false);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
</script>
|
||||||
|
{% endblock %}
|
59
migrator/templates/migration_progress.html
Normal file
59
migrator/templates/migration_progress.html
Normal file
|
@ -0,0 +1,59 @@
|
||||||
|
{% extends 'base.html' %}
|
||||||
|
|
||||||
|
{% block content %}
|
||||||
|
<h1>Migration progress for {{ migration.username }}</h1>
|
||||||
|
|
||||||
|
<div class="mt-4">
|
||||||
|
<ul class="list-group">
|
||||||
|
{% for repository in migration.repositories.all %}
|
||||||
|
<li id="revision-{{ repository.id }}" class="list-group-item">
|
||||||
|
<div class="d-flex flex-row">
|
||||||
|
<div class="d-inline-block mr-auto">
|
||||||
|
{{ repository.path_with_namespace }}
|
||||||
|
</div>
|
||||||
|
<div class="d-inline-block">
|
||||||
|
<span class="status status-{{ repository.result }}">
|
||||||
|
</span>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<pre style="height:140px" class="error_msg form-control is-invalid mt-2 mb-0{% if repository.result != 4 %} d-none{% endif %}">{{ repository.error_message }}</pre>
|
||||||
|
</li>
|
||||||
|
{% endfor %}
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
{% endblock %}
|
||||||
|
|
||||||
|
{% block javascript %}
|
||||||
|
<style>
|
||||||
|
.status-1::after { content: "pending"; color: #998800; }
|
||||||
|
.status-2::after { content: "success"; color: #339933; font-weight: bold; }
|
||||||
|
.status-3::after { content: "exists"; color: #666666; }
|
||||||
|
.status-4::after { content: "error"; color: #992222; }
|
||||||
|
.status-5::after { content: "in progress"; color: #998800; }
|
||||||
|
</style>
|
||||||
|
<script type="text/javascript">
|
||||||
|
$(document).ready(function() {
|
||||||
|
var PENDING = 1;
|
||||||
|
var intervalId;
|
||||||
|
function updateStatus() {
|
||||||
|
$.getJSON("{% url 'migration_status' migration.id %}", function(status) {
|
||||||
|
var alldone = true;
|
||||||
|
|
||||||
|
status.forEach(function (item) {
|
||||||
|
if(item.status == PENDING) { alldone = false; }
|
||||||
|
$('#revision-'+item.id+' span.status').attr('class','status status-'+item.status);
|
||||||
|
if(typeof(item.message) !== "undefined") {
|
||||||
|
$('#revision-'+item.id+' .error_msg').text(item.message).removeClass('d-none');
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
if(alldone) {
|
||||||
|
clearInterval(intervalId);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
intervalId = setInterval(updateStatus, 10000);
|
||||||
|
});
|
||||||
|
</script>
|
||||||
|
{% endblock %}
|
3
migrator/tests.py
Normal file
3
migrator/tests.py
Normal file
|
@ -0,0 +1,3 @@
|
||||||
|
from django.test import TestCase
|
||||||
|
|
||||||
|
# Create your tests here.
|
12
migrator/urls.py
Normal file
12
migrator/urls.py
Normal file
|
@ -0,0 +1,12 @@
|
||||||
|
from django.urls import path, include
|
||||||
|
from . import views
|
||||||
|
|
||||||
|
urlpatterns = [
|
||||||
|
path('api/', include([
|
||||||
|
path('gitlab/projects', views.fetch_gitlab_projects, name="fetch_gitlab_projects"),
|
||||||
|
path('gitea/access', views.check_gitea_access, name="check_gitea_access"),
|
||||||
|
path('migration/status/<str:migration_key>', views.migration_status, name="migration_status"),
|
||||||
|
])),
|
||||||
|
path('', views.migration_create, name="migration_create"),
|
||||||
|
path('progress/<str:migration_key>', views.migration_progess, name="migration_progress"),
|
||||||
|
]
|
117
migrator/views.py
Normal file
117
migrator/views.py
Normal file
|
@ -0,0 +1,117 @@
|
||||||
|
from django.shortcuts import render, redirect
|
||||||
|
from django.http import JsonResponse, HttpResponse
|
||||||
|
from . import settings
|
||||||
|
from .helper import get_dict_from_query
|
||||||
|
from .models import *
|
||||||
|
from . import tasks
|
||||||
|
from .api import *
|
||||||
|
|
||||||
|
def migration_create(request):
|
||||||
|
if request.method == "POST":
|
||||||
|
projects = get_dict_from_query(request.POST,"projects")
|
||||||
|
|
||||||
|
migration = Migration(
|
||||||
|
gitlab_token = request.POST["gitlab_token"],
|
||||||
|
gitea_token = request.POST["gitea_token"],
|
||||||
|
username = request.POST["username"],
|
||||||
|
)
|
||||||
|
|
||||||
|
migration.save()
|
||||||
|
|
||||||
|
for project in projects:
|
||||||
|
repo = Repository(
|
||||||
|
migration = migration,
|
||||||
|
path_with_namespace = project
|
||||||
|
)
|
||||||
|
repo.save()
|
||||||
|
tasks.migrate_repository(repo.id)
|
||||||
|
|
||||||
|
return redirect("migration_progress", migration.id)
|
||||||
|
|
||||||
|
return render(request, "migration_create.html")
|
||||||
|
|
||||||
|
def migration_progess(request, migration_key):
|
||||||
|
try:
|
||||||
|
migration = Migration.objects.prefetch_related('repositories').get(pk=migration_key)
|
||||||
|
except Migration.DoesNotExist:
|
||||||
|
return redirect("migration_create")
|
||||||
|
|
||||||
|
return render(request, "migration_progress.html", {
|
||||||
|
'migration': migration,
|
||||||
|
})
|
||||||
|
|
||||||
|
def migration_status(request, migration_key):
|
||||||
|
repositories = Repository.objects.filter(
|
||||||
|
migration_id = migration_key,
|
||||||
|
).all()
|
||||||
|
|
||||||
|
data = []
|
||||||
|
for repo in repositories:
|
||||||
|
info = { 'id': repo.id, 'status': repo.result }
|
||||||
|
if repo.result == Status.ERROR:
|
||||||
|
info['message'] = repo.error_message
|
||||||
|
data.append(info)
|
||||||
|
|
||||||
|
return JsonResponse(data,safe=False)
|
||||||
|
|
||||||
|
def fetch_gitlab_projects(request):
|
||||||
|
gitlab_token = request.GET.get("gitlab_token", None)
|
||||||
|
if gitlab_token is None:
|
||||||
|
return JsonResponse({ 'error' : 'Missing token' })
|
||||||
|
|
||||||
|
gitlabapi = GitlabAPI(gitlab_token)
|
||||||
|
|
||||||
|
response = gitlabapi.get('/user')
|
||||||
|
|
||||||
|
if response.status_code != 200:
|
||||||
|
return JsonResponse({ 'error' : 'API error' })
|
||||||
|
|
||||||
|
data = response.json()
|
||||||
|
|
||||||
|
try:
|
||||||
|
username = data['username']
|
||||||
|
except KeyError:
|
||||||
|
return JsonResponse({ 'error' : 'API error' })
|
||||||
|
|
||||||
|
projects = []
|
||||||
|
|
||||||
|
page = 1
|
||||||
|
while True:
|
||||||
|
response = gitlabapi.get('/projects', params={
|
||||||
|
'owned': 1,
|
||||||
|
'page': page,
|
||||||
|
'per_page': 100,
|
||||||
|
})
|
||||||
|
|
||||||
|
if response.status_code != 200:
|
||||||
|
return JsonResponse({ 'error' : 'API error' })
|
||||||
|
|
||||||
|
data = response.json()
|
||||||
|
|
||||||
|
if len(data) == 0:
|
||||||
|
break
|
||||||
|
|
||||||
|
for gitlab_project in data:
|
||||||
|
try:
|
||||||
|
projects.append(gitlab_project['path_with_namespace'])
|
||||||
|
except KeyError:
|
||||||
|
return JsonResponse({ 'error' : 'API error' })
|
||||||
|
|
||||||
|
page+=1
|
||||||
|
|
||||||
|
return JsonResponse({
|
||||||
|
'username' : username,
|
||||||
|
'projects' : sorted(projects)
|
||||||
|
})
|
||||||
|
|
||||||
|
def check_gitea_access(request):
|
||||||
|
gitea_token = request.GET.get("gitea_token", None)
|
||||||
|
if gitea_token is None:
|
||||||
|
return JsonResponse({ 'success': False })
|
||||||
|
|
||||||
|
response = GiteaAPI(gitea_token).get('/user')
|
||||||
|
|
||||||
|
if response.status_code == 200:
|
||||||
|
return JsonResponse({ 'success': True })
|
||||||
|
else:
|
||||||
|
return JsonResponse({ 'success': False })
|
7
static/bootstrap.min.css
vendored
Normal file
7
static/bootstrap.min.css
vendored
Normal file
File diff suppressed because one or more lines are too long
2
static/jquery-3.5.1.min.js
vendored
Normal file
2
static/jquery-3.5.1.min.js
vendored
Normal file
File diff suppressed because one or more lines are too long
Loading…
Reference in a new issue