Merge pull request #715 from meolu/feature/2.0-local-git

walle 2.1 - fix: #627 优化上线的branch/tag/commit获取，稳定且提速获取

walle/app.py

@@ -1,8 +1,10 @@
 # -*- coding: utf-8 -*-
 """The app module, containing the app factory function."""
+import gevent.monkey
+gevent.monkey.patch_all()
+
 import logging
 import sys
-
 import os
 from flask import Flask, render_template, current_app
 from flask_restful import Api

walle/service/deployer.py

@@ -22,6 +22,7 @@ from walle.service.utils import color_clean
 from walle.service.utils import excludes_format, includes_format
 from walle.service.notice import Notice
 from walle.service.waller import Waller
+from walle.service.git.repo import Repo
 from flask_login import current_user
 
 class Deployer:
@@ -395,77 +396,23 @@ class Deployer:
         return errors
 
     def list_tag(self):
-        self.init_repo()
-
-        with self.localhost.cd(self.dir_codebase_project):
-            command = 'git tag -l'
-            result = self.localhost.local(command, pty=False, wenv=self.config())
-            tags = result.stdout.strip()
-            tags = tags.split('\n')
-            return [color_clean(tag.strip()) for tag in tags]
+        repo = Repo(self.dir_codebase_project)
+        repo.init(url=self.project_info['repo_url'])
 
-        return []
+        return repo.tags()
 
     def list_branch(self):
-        self.init_repo()
-
-        with self.localhost.cd(self.dir_codebase_project):
-            command = 'git pull'
-            result = self.localhost.local(command, wenv=self.config())
-
-            if result.exited != Code.Ok:
-                raise WalleError(Code.shell_git_pull_fail, message=result.stdout)
-
-            current_app.logger.info(self.dir_codebase_project)
-
-            command = 'git branch -r'
-            result = self.localhost.local(command, pty=False, wenv=self.config())
-
-            # if result.exited != Code.Ok:
-            #     raise WalleError(Code.shell_run_fail)
+        repo = Repo(self.dir_codebase_project)
+        repo.init(url=self.project_info['repo_url'])
 
-            # TODO 三种可能: false, error, success
-            branches = result.stdout.strip()
-            branches = branches.split('\n')
-            # 去除 origin/HEAD -> 当前指向
-            # 去除远端前缀
-            branches = [branch.strip().lstrip('origin/') for branch in branches if
-                        not branch.strip().startswith('origin/HEAD')]
-            return branches
-
-        return None
+        return repo.branches()
 
     def list_commit(self, branch):
-        self.init_repo()
-        with self.localhost.cd(self.dir_codebase_project):
-            command = 'git checkout %s && git pull' % (branch)
-            self.localhost.local(command, wenv=self.config())
-
-            command = 'git log -50 --pretty="%h #@_@# %an #@_@# %s"'
-            result = self.localhost.local(command, pty=False, wenv=self.config())
-            current_app.logger.info(result.stdout)
-
-            commit_log = result.stdout.strip()
-            current_app.logger.info(commit_log)
-            commit_list = commit_log.split('\n')
-            commits = []
-            for commit in commit_list:
-                if not re.search('^.+ #@_@# .+ #@_@# .*$', commit):
-                    continue
-
-                commit_dict = commit.split(' #@_@# ')
-                current_app.logger.info(commit_dict)
-                commits.append({
-                    'id': commit_dict[0],
-                    'name': commit_dict[1],
-                    'message': commit_dict[2],
-                })
-
-            return commits
-
-        # TODO
-        return None
+        repo = Repo(self.dir_codebase_project)
+        repo.init(url=self.project_info['repo_url'])
+        return repo.commits(branch)
 
+    # 待废弃，迁移到gitpython
     def init_repo(self):
         if not os.path.exists(self.dir_codebase_project):
             # 检查 目录是否存在
@@ -481,6 +428,7 @@ class Deployer:
             command = 'rm -rf %s' % (self.dir_codebase_project)
             self.localhost.local(command, wenv=self.config())
 
+            # 切换到gitpython模式
             command = 'git clone %s %s' % (self.project_info['repo_url'], self.dir_codebase_project)
             current_app.logger.info('cd %s  command: %s  ' % (self.dir_codebase_project, command))
 

walle/service/git/__init__.py

@@ -0,0 +1,8 @@
+# -*- coding: utf-8 -*-
+"""
+    walle-web
+
+    :copyright: © 2015-2019 walle-web.io
+    :created time: 2019-02-24 10:47:28
+    :author: wushuiyong@walle-web.io
+"""

walle/service/git/repo.py

@@ -0,0 +1,132 @@
+# -*- coding: utf-8 -*-
+"""
+    walle-web
+
+    :copyright: © 2015-2019 walle-web.io
+    :created time: 2019-02-24 10:47:53
+    :author: wushuiyong@walle-web.io
+"""
+
+import os
+import re
+import os.path as osp
+import git as PyGit
+from git import Repo as PyRepo
+
+
+class Repo:
+    path = None
+
+    def __init__(self, path=None):
+        self.path = path
+
+    def is_git_dir(self):
+        '''
+        判断是否为git目录
+
+        @param path:
+        @return:
+        '''
+        d = self.path + '/.git'
+        if osp.isdir(d):
+            if osp.isdir(osp.join(d, 'objects')) and osp.isdir(osp.join(d, 'refs')):
+                headref = osp.join(d, 'HEAD')
+                return osp.isfile(headref) or \
+                       (osp.islink(headref) and
+                        os.readlink(headref).startswith('refs'))
+            elif (osp.isfile(osp.join(d, 'gitdir')) and
+                  osp.isfile(osp.join(d, 'commondir')) and
+                  osp.isfile(osp.join(d, 'gitfile'))):
+                return False
+        return False
+
+    def init(self, url):
+        # 创建目录
+        if not os.path.exists(self.path):
+            os.makedirs(self.path)
+        # git clone
+        if self.is_git_dir():
+            return self.pull()
+        else:
+            return self.clone(url)
+
+    def clone(self, url):
+        '''
+        检出项目
+
+        @param branch:
+        @param kwargs:
+        @return:
+        '''
+        return PyRepo.clone_from(url, self.path)
+
+    def pull(self):
+        '''
+        更新项目
+
+        @param branch:
+        @param kwargs:
+        @return:
+        '''
+        repo = PyRepo(self.path)
+
+        return repo.remote().pull()
+
+    def checkout_2_branch(self, branch):
+        PyRepo(self.path).git.checkout(branch)
+
+    def checkout_2_commit(self, branch, commit):
+        PyRepo(self.path).head.set_reference(branch)
+        PyRepo(self.path).head.set_commit(commit)
+
+    def branches(self):
+        '''
+        获取所有分支
+
+        @param branch:
+        @param kwargs:
+        @return:
+        '''
+        # 去除 origin/HEAD -> 当前指向
+        # 去除远端前缀
+        branches = PyRepo(self.path).remote().refs
+        return [str(branch).strip().lstrip('origin/') for branch in branches if
+                not str(branch).strip().startswith('origin/HEAD')]
+
+    def tags(self):
+        '''
+        获取所有tag
+
+        @param branch:
+        @param kwargs:
+        @return:
+        '''
+        return [str(tag) for tag in PyRepo(self.path).tags]
+
+    def commits(self, branch):
+        '''
+        获取分支的commits
+
+        @param branch:
+        @param kwargs:
+        @return:
+        '''
+        self.checkout_2_branch(branch)
+
+        commit_log = PyGit.Git(self.path).log('--pretty=%h #@_@# %an #@_@# %s', max_count=50)
+        commit_list = commit_log.split('\n')
+        commits = []
+        for commit in commit_list:
+            if not re.search('^.+ #@_@# .+ #@_@# .*$', commit):
+                continue
+
+            commit_dict = commit.split(' #@_@# ')
+            from flask import current_app
+            current_app.logger.info(commit_dict)
+            commits.append({
+                'id': commit_dict[0],
+                'name': commit_dict[1],
+                'message': commit_dict[2],
+            })
+
+        return commits

walle/service/waller.py

@@ -88,10 +88,15 @@ class Waller(Connection):
             return result
 
         except Exception as e:
+            # TODO 貌似可能的异常有很多种，需要分层才能完美解决 something wrong without e.result
             current_app.logger.exception(e)
+            if hasattr(e, 'message'):
+                error = e.message
+            elif hasattr(e, 'result'):
+                error = e.result
+            else:
+                error = str(e)
 
-            # TODO 貌似可能的异常有很多种，需要分层才能完美解决 something wrong without e.result
-            error = e.result if 'result' in e else e.message
             RecordModel().save_record(stage=wenv['stage'], sequence=wenv['sequence'], user_id=wenv['user_id'],
                                       task_id=wenv['task_id'], status=1, host=self.host, user=self.user,
                                       command=command, success='', error=error)
@@ -101,7 +106,7 @@ class Waller(Connection):
                 )
             else:
                 message = 'task_id=%s, user:%s host:%s command:%s, status=1, message:%s' % (
-                    wenv['task_id'], self.user, self.host, command, e.message
+                    wenv['task_id'], self.user, self.host, command, error
                 )
             current_app.logger.error(message, exc_info=1)