walle 2.0 alpha - logs

admin.sh

@@ -20,7 +20,7 @@ function init() {
     virtualenv --no-site-packages venv # 注意:安装失败请指定python路径. mac 可能会有用anaconda的python
     source ./venv/bin/activate
     pip install -r ./requirements/prod.txt
-    echo "****************"
+    echo "************************************************"
     echo -e "\033[32m init walle success \033[0m"
     echo -e "\033[32m welcome to walle 2.0 \033[0m"
 }
@@ -74,39 +74,39 @@ function migration() {
 
 case "$1" in
     init )
-        echo "****************"
+        echo "************************************************"
         init
-        echo "****************"
+        echo "************************************************"
         ;;
     start )
-        echo "****************"
+        echo "************************************************"
         start
-        echo "****************"
+        echo "************************************************"
         ;;
     stop )
-        echo "****************"
+        echo "************************************************"
         stop
-        echo "****************"
+        echo "************************************************"
         ;;
     restart )
-        echo "****************"
+        echo "************************************************"
         restart
-        echo "****************"
+        echo "************************************************"
         ;;
     upgrade )
-        echo "****************"
+        echo "************************************************"
         upgrade
         migration
-        echo "****************"
+        echo "************************************************"
         ;;
     migration )
-        echo "****************"
+        echo "************************************************"
         migration
-        echo "****************"
+        echo "************************************************"
         ;;
     * )
-        echo "****************"
+        echo "************************************************"
         echo "Usage: sh admin {init|start|stop|restart|upgrade|migration}"
-        echo "****************"
+        echo "************************************************"
         ;;
 esac

walle/app.py

@@ -126,19 +126,10 @@ def register_errorhandlers(app):
 
     @app.errorhandler(WalleError)
     def render_error(error):
-        app.logger.info('============ register_errorhandlers ============')
         # response 的 json 内容为自定义错误代码和错误信息
+        app.logger.error(error, exc_info=1)
         return error.render_error()
 
-    def render_errors():
-        """Render error template."""
-        app.logger.info('============ render_errors ============')
-        # If a HTTPException, pull the `code` attribute; default to 500
-        return ApiResource.render_json(code=Code.space_error)
-        #
-        #     error_code = getattr(error, 'code', 500)
-        #     return render_template('{0}.html'.format(error_code)), error_code
-
 
 def register_shellcontext(app):
     """Register shell context objects."""

walle/service/code.py

@@ -9,6 +9,11 @@
 
 
 class Code():
+
+    #: 没有消息就是好消息
+    Ok = 0
+
+    #: ----------------------- 1xxx 用户相关错误 -----------------
     #: 1xxx 表示用户相关: 登录, 权限
     #: 未登录, 大概是永远不会变了
     unlogin = 1000
@@ -28,6 +33,7 @@ class Code():
     #: 用户不存在
     user_not_in_space = 1005
 
+    #: ----------------------- 2xxx 前端相关错误 -----------------
     #: 2xxx 前端相关错误
     #: 参数错误
     params_error = 2000
@@ -35,14 +41,36 @@ class Code():
     #: 表单错误
     form_error = 2001
 
+    #: ----------------------- 3xxx shell 相关错误 -----------------
+    #: 3xxx shell相关错误
+    #: 不知道怎么归类的错误
+    shell_run_fail = 3000
+
+    #: 目录不存在
+    shell_dir_not_exists = 3001
+
+    #: ----------------------- 4xxx git 相关错误 -----------------
+    #: 3xxx shell相关错误
+    #: git尚未初始化
+    shell_git_init_fail = 4001
+
+    #: git pull 失败
+    shell_git_pull_fail = 4002
 
     code_msg = {
         unlogin: '未登录',
         error_pwd: '账号密码错误',
         not_allow: '无此资源权限',
-        params_error: '参数错误',
         space_empty: '尚未开通空间，请联系空间负责人加入空间',
         space_error: '无此空间权限',
         user_not_in_space: '用户不存在此空间，请联系空间所有人添加此用户到用户组',
+
+        params_error: '参数错误',
         form_error: '表单错误',
+
+        shell_run_fail: '命令运行错误，请联系管理员',
+        shell_dir_not_exists: '路径不存在，请联系管理员',
+
+        shell_git_init_fail: '项目git初始化失败，请联系管理员',
+        shell_git_pull_fail: 'git pull 失败，请联系管理员',
     }

walle/service/deployer.py

@@ -9,20 +9,19 @@ import time
 from datetime import datetime
 
 import os
+import re
 from flask import current_app
-
-from walle.model.record import RecordModel
-from walle.service.waller import Waller
+from flask_socketio import emit
 from walle.model.project import ProjectModel
+from walle.model.record import RecordModel
 from walle.model.task import TaskModel
-from flask_socketio import emit
 from walle.service.extensions import socketio
 from walle.service.utils import color_clean
-import re
-
+from walle.service.waller import Waller
+from walle.service.code import Code
+from walle.service.error import WalleError
 
 class Deployer:
-
     '''
     序列号
     '''
@@ -80,9 +79,9 @@ class Deployer:
         # start to deploy
         self.console = console
 
-
     def config(self):
-        return {'task_id': self.task_id, 'user_id': self.user_id, 'stage': self.stage, 'sequence': self.sequence, 'console': self.console}
+        return {'task_id': self.task_id, 'user_id': self.user_id, 'stage': self.stage, 'sequence': self.sequence,
+                'console': self.console}
 
     def start(self):
         TaskModel().get_by_id(self.task_id).update({'status': TaskModel.status_doing})
@@ -132,7 +131,6 @@ class Deployer:
         with self.local.cd(self.dir_codebase_project):
             result = self.local.run(command, wenv=self.config())
 
-
     def deploy(self):
         '''
         2.检出代码
@@ -231,7 +229,6 @@ class Deployer:
         with waller.cd(self.project_info['target_releases']):
             result = waller.run(command, wenv=self.config())
 
-
         # TODO md5
         # 传送到版本库 release
         current_app.logger.info('/tmp/walle/codebase/' + self.release_version_tar)
@@ -313,36 +310,36 @@ class Deployer:
 
         with self.local.cd(self.dir_codebase_project):
             command = 'git tag -l'
-            current_app.logger.info('cd %s  command: %s  ', self.dir_codebase_project, command)
             result = self.local.run(command, wenv=self.config())
-            current_app.logger.info(dir(result))
             return result
 
         return None
 
     def list_branch(self):
         self.init_repo()
-        socketio.sleep(61)
 
         with self.local.cd(self.dir_codebase_project):
             command = 'git pull'
-
-            from flask import current_app
-            from walle.service import utils
-            current_app.logger.info(utils.detailtrace())
             result = self.local.run(command, wenv=self.config())
 
+            if result.exited != Code.Ok:
+                raise WalleError(Code.shell_git_pull_fail)
+
             current_app.logger.info(self.dir_codebase_project)
 
             command = 'git branch -r'
             result = self.local.run(command, wenv=self.config())
 
+            if result.exited != Code.Ok:
+                raise WalleError(Code.shell_run_fail)
+
             # TODO 三种可能: false, error, success
             branches = color_clean(result.stdout.strip())
             branches = branches.split('\n')
             # 去除 origin/HEAD -> 当前指向
             # 去除远端前缀
-            branches = [branch.strip().lstrip('origin/') for branch in branches if not branch.strip().startswith('origin/HEAD')]
+            branches = [branch.strip().lstrip('origin/') for branch in branches if
+                        not branch.strip().startswith('origin/HEAD')]
             return branches
 
         return None
@@ -355,11 +352,10 @@ class Deployer:
             result = self.local.run(command, wenv=self.config())
 
             # TODO 10是需要前端传的
-            command = 'git log -10 --pretty="%h #_# %an #_# %s"'
+            command = 'git log -50 --pretty="%h #_# %an #_# %s"'
             result = self.local.run(command, wenv=self.config())
-            current_app.logger.info(result.stdout.strip())
+
             commit_log = color_clean(result.stdout.strip())
-            current_app.logger.info(commit_log)
             commit_list = commit_log.split('\n')
             commits = []
             for commit in commit_list:
@@ -379,24 +375,23 @@ class Deployer:
         return None
 
     def init_repo(self):
-
-        current_app.logger.info('git dir: %s', self.dir_codebase_project + '/.git')
-        # 如果项目底下有 .git 目录则认为项目完整,可以直接检出代码
-        # TODO 不标准
         if not os.path.exists(self.dir_codebase_project):
             # 检查 目录是否存在
             command = 'mkdir -p %s' % (self.dir_codebase_project)
             # TODO remove
             current_app.logger.info(command)
-            result = self.local.run(command, wenv=self.config())
+            self.local.run(command, wenv=self.config())
 
-        if not os.path.exists(self.dir_codebase_project + '/.git'):
+        is_git_dir = self.local.run('git status', wenv=self.config())
+        if is_git_dir.exited != Code.Ok:
             # 否则当作新项目检出完整代码
             with self.local.cd(self.dir_codebase_project):
-                command = 'pwd && git clone %s .' % (self.project_info['repo_url'])
+                command = 'pwd && rm -rf .* .git && git clone %s .' % (self.project_info['repo_url'])
                 current_app.logger.info('cd %s  command: %s  ', self.dir_codebase_project, command)
 
                 result = self.local.run(command, wenv=self.config())
+                if result.exited != Code.Ok:
+                    raise WalleError(Code.shell_git_init_fail)
 
     def end(self, success=True):
         status = TaskModel.status_success if success else TaskModel.status_fail
@@ -417,53 +412,9 @@ class Deployer:
                 self.release(self.connections[server])
                 self.post_release(self.connections[server])
             except Exception as e:
+                current_app.logger.error(e)
                 all_servers_success = False
                 self.errors[server] = e.message
 
         self.end(all_servers_success)
         return {'success': self.success, 'errors': self.errors}
-
-    def test(self):
-        # server = '172.20.95.43'
-        server = '172.16.0.231'
-        ws_dict = {
-                'user': 'xx',
-                'host': 'ip',
-                'cmd': 'Going to sleep !!!!!',
-                'status': 0,
-                'stage': 1,
-                'sequence': 1,
-                'success': 'Going to sleep !!!!!',
-                'error': '',
-            }
-        emit('console', {'event': 'task:console', 'data': ws_dict}, room=self.task_id)
-        socketio.sleep(60)
-        ws_dict = {
-                'user': 'xx',
-                'host': 'ip',
-                'cmd': 'sleep 60',
-                'status': 0,
-                'stage': 1,
-                'sequence': 1,
-                'success': 'sleep 60....',
-                'error': '',
-            }
-        emit('console', {'event': 'task:console', 'data': ws_dict}, room=self.task_id)
-        socketio.sleep(10)
-        ws_dict = {
-                'user': 'xx',
-                'host': 'ip',
-                'cmd': 'sleep 10',
-                'status': 0,
-                'stage': 1,
-                'sequence': 1,
-                'success': 'sleep 10....',
-                'error': '',
-            }
-        emit('console', {'event': 'task:console', 'data': ws_dict}, room=self.task_id)
-        try:
-            self.connections[server] = Waller(host=server, user='work')
-            self.post_release_service(self.connections[server])
-        except Exception as e:
-            current_app.logger.exception(e)
-            self.errors[server] = e.message

walle/service/error.py

@@ -30,6 +30,7 @@ class WalleError(Exception):
         if self.code not in Code.code_msg:
             current_app.logger.error('unkown code %s' % (self.code))
 
+        current_app.logger.error(self, exc_info=1)
         if self.code in Code.code_msg:
             self.message = Code.code_msg[self.code]
 

walle/service/extensions.py

@@ -19,4 +19,4 @@ login_manager = LoginManager()
 mail = Mail()
 
 permission = Permission()
-socketio = SocketIO()
+socketio = SocketIO(engineio_logger=True, logger=True)

walle/service/waller.py

@@ -8,7 +8,8 @@ from fabric2 import Connection
 from flask import current_app
 from flask_socketio import emit
 from walle.model.record import RecordModel
-
+from invoke import Result
+from walle.service.code import Code
 
 class Waller(Connection):
     connections, success, errors = {}, {}, {}
@@ -41,7 +42,6 @@ class Waller(Connection):
             message = 'task_id=%s, host:%s command:%s status:%s, success:%s, error:%s' % (
                 wenv['task_id'], self.host, command, exitcode, stdout, stderr
             )
-            current_app.logger.error(result.stdout.strip())
             # TODO
             ws_dict = {
                 'user': self.user,
@@ -60,18 +60,17 @@ class Waller(Connection):
                                       task_id=wenv['task_id'], status=exitcode, host=self.host, user=self.user,
                                       command=result.command, success=stdout,
                                       error=stderr)
-            current_app.logger.info(message)
-            if exitcode != 0:
-                # TODO
+            current_app.logger.info(result)
+            if exitcode != Code.Ok:
+                current_app.logger.exception(result.stdout.strip())
                 return result
             return result
 
         except Exception as e:
-            # current_app.logger.exception(e)
-            # return None
+            current_app.logger.exception(e)
+
             # TODO 貌似可能的异常有很多种，需要分层才能完美解决 something wrong without e.result
             error = e.result if 'result' in e else e.message
-            current_app.logger.error(e)
             RecordModel().save_record(stage=wenv['stage'], sequence=wenv['sequence'], user_id=wenv['user_id'],
                                       task_id=wenv['task_id'], status=1, host=self.host, user=self.user,
                                       command=command, success='', error=error)
@@ -83,6 +82,7 @@ class Waller(Connection):
                 message = 'task_id=%s, user:%s host:%s command:%s, status=1, message:%s' % (
                     wenv['task_id'], self.user, self.host, command, e.message
                 )
+            current_app.logger.error(message, exc_info=1)
 
             # TODO
             ws_dict = {
@@ -93,13 +93,13 @@ class Waller(Connection):
                 'stage': wenv['stage'],
                 'sequence': wenv['sequence'],
                 'success': '',
-                'error': e.message,
+                'error': error,
             }
             if wenv['console']:
                 emit('console', {'event': 'task:console', 'data': ws_dict}, room=wenv['task_id'])
-            current_app.logger.error(message)
 
-            return False
+            return Result(exited=-1, stderr=error)
+
 
     def sudo(self, command, wenv=None, **kwargs):
         return self.run(command, wenv=wenv, sudo=True, **kwargs)

walle/templates/socketio.html

@@ -13,8 +13,18 @@
     welcome to walle 2.0
 </div>
 <br>
-<input id="say" type="text">
-<button id="send" type="button">Send!</button>
+
+<button id="branch" type="button">Branch</button>
+<br>
+
+<button id="tag" type="button">Tag</button>
+<br>
+
+<button id="commit" type="button">Commit</button>
+<br>
+
+<input id="say" type="text"><button id="send" type="button">Send!</button>
+
 <body>
 <script src="http://code.jquery.com/jquery-1.8.2.min.js"></script>
 <script type="text/javascript" src="/socket.io.min.js"></script>
@@ -95,15 +105,27 @@
             solo.html(solo.html() + '<br>' + msg);
         }
 
-        window.setInterval(function() {
-            start_time = (new Date).getTime();
-            socket.emit('ping', {'start_time': start_time});
-        }, 2000);
+{#        window.setInterval(function() {#}
+{#            start_time = (new Date).getTime();#}
+{#            socket.emit('ping', {'start_time': start_time});#}
+{#        }, 10000);#}
 
         socket.on('pong', function(data) {
             write(data)
         });
 
+        $('#branch').click(function(e) {
+            socket.emit('branches', {});
+        })
+
+        $('#tag').click(function(e) {
+            socket.emit('tags', {});
+        })
+
+        $('#commit').click(function(e) {
+            socket.emit('commits', {'branch': 'master'});
+        })
+
         $(document).keyup(function (event) {
             if (event.keyCode == 13) {
                 $("#send").trigger("click");