一、实际生产的架构图
1、生产环境为什么要这样干
- 解耦
- 异步
2、常用的queue软件
- redis, rabbitmq
- github ,
- restful API
- celery
二、我们今天如何实现？
1、实现思路
问题：views和web之间已返回，线程这里就断开了，那是因为你用django又启了一个线程
怎样才启动一个独立的进程，和django是没有关系，只不过是用django启动的，由操作系统来管理
三、目录结构
四、代码实现
1、backend
1、main
import subprocess
from web import models
from django.contrib.auth import authenticate
import random,string,uuid
class HostManager(object):
"""用户登陆堡垒机后的交互程序"""
def __init__(self):
self.user = None
def get_session_id(self,bind_host_obj,tag):
'''apply session id'''
session_obj = models.Session(user_id = self.user.id,bind_host=bind_host_obj,tag=tag)
session_obj.save()
return session_obj
def interactive(self):
"""交互脚本"""
print("----run---------")
count = 0
while count <3:
username = input("Username:").strip()
password = input("Password:").strip()
user = authenticate(username=username,password=password)
if user:
print("Welcome %s".center(50,'-') % user.name )
self.user = user
break
else:
print("Wrong username or password!")
count += 1
else:
exit("Too many attempts, bye.")
if self.user: #验证成功
while True:
for index,host_group in enumerate(self.user.host_groups.all()): #select_related()
print("%s.\t%s[%s]" %(index,host_group.name, host_group.bind_hosts.count()))
print("z.\t未分组主机[%s]" %(self.user.bind_hosts.count()))
choice = input("%s>>:"% self.user.name).strip()
if len(choice) == 0:continue
selected_host_group = None
if choice.isdigit():
choice = int(choice)
if choice >=0 and choice <= index: #合法选项
selected_host_group = self.user.host_groups.all()[choice]
elif choice == 'z':
selected_host_group = self.user
if selected_host_group:
print("selected host group", selected_host_group)
while True:
for index, bind_host in enumerate(selected_host_group.bind_hosts.all()):
print("%s.\t%s" % (index, bind_host))
choice = choice = input("%s>>>:" % self.user.name).strip()
if choice.isdigit():
choice = int(choice)
if choice >= 0 and choice <= index: # 合法选项
print("going to logon ....", selected_host_group.bind_hosts.all()[choice])
bind_host = selected_host_group.bind_hosts.all()[choice]
ssh_tag = uuid.uuid4()
session_obj = self.get_session_id(bind_host,ssh_tag)
monitor_script = subprocess.Popen("sh /opt/CrazyEye/backend/session_tracker.sh %s %s" % (ssh_tag,session_obj.id),shell=True,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
#print(monitor_script.stderr.read())
subprocess.run('sshpass -p %s ssh %s@%s -E %s -o StrictHostKeyChecking=no' %(bind_host.remote_user.password,
bind_host.remote_user.username,
bind_host.host.ip_addr ,ssh_tag ), shell=True)
elif choice == 'b':
break
2、task_manager
import json,os ,subprocess
from django import conf
from web import models
class MultiTaskManger(object):
"""负责解析并触发批量任务"""
def __init__(self,request):
self.request = request
self.call_task()
def task_parser(self):
"""解析任务"""
self.task_data = json.loads(self.request.POST.get("task_data"))
def call_task(self):
self.task_parser()
if self.task_data['task_type'] == 0:#cmd
self.cmd_task()
elif self.task_data['task_type'] == 1:#file transfer
self.file_transfer_task()
def cmd_task(self):
"""
1.生产任务id
2.触发任务
3.返回任务id
:return:
"""
task_obj = models.Task.objects.create(user=self.request.user,
task_type=self.task_data['task_type'],
content = self.task_data["cmd"])
sub_task_objs = []
for host_id in self.task_data['selected_host_ids'] :
sub_task_objs.append(models.TaskLogDetail(task=task_obj,bind_host_id=host_id,result='init...',status=2))
models.TaskLogDetail.objects.bulk_create(sub_task_objs)
task_script_obj = subprocess.Popen("python %s %s" %(conf.settings.MULTITASK_SCRIPT,task_obj.id),
shell=True,stdout=subprocess.PIPE)
self.task = task_obj
def file_transfer_task(self):
"""
1.生产任务记录
2.触发任务
3. 返回任务id
:return:
"""
task_obj = models.Task.objects.create(user=self.request.user,
task_type=self.task_data['task_type'],
content=json.dumps(self.task_data))
sub_task_objs = []
for host_id in self.task_data['selected_host_ids']:
sub_task_objs.append(models.TaskLogDetail(task=task_obj, bind_host_id=host_id, result='init...', status=2))
models.TaskLogDetail.objects.bulk_create(sub_task_objs)
task_script_obj = subprocess.Popen("python %s %s" % (conf.settings.MULTITASK_SCRIPT, task_obj.id),
shell=True, stdout=subprocess.PIPE)
self.task = task_obj
3、task_runner
import sys ,os
import time,json
from concurrent.futures import ThreadPoolExecutor
import paramiko
def ssh_cmd(task_log_obj):
host = task_log_obj.bind_host.host
user_obj = task_log_obj.bind_host.remote_user
try:
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect(host.ip_addr, host.port, user_obj.username, user_obj.password,timeout=10)
stdin, stdout, stderr = ssh.exec_command(task_log_obj.task.content)
stdout_res = stdout.read()
stderr_res = stderr.read()
result = stdout_res + stderr_res
print(result)
task_log_obj.result = result
task_log_obj.status = 0
ssh.close()
except Exception as e :
task_log_obj.result = e
task_log_obj.status = 1
task_log_obj.save()
def file_transfer(task_log_obj):
host = task_log_obj.bind_host.host
user_obj = task_log_obj.bind_host.remote_user
try:
t = paramiko.Transport((host.ip_addr, host.port))
t.connect(username=user_obj.username, password=user_obj.password)
sftp = paramiko.SFTPClient.from_transport(t)
task_data = json.loads(task_log_obj.task.content)
if task_data['file_transfer_type'] == 'send':
sftp.put(task_data['local_file_path'],task_data['remote_file_path'])
task_log_obj.result = "send local file [%s] to remote [%s] succeeded!" %(task_data['local_file_path'],
task_data['remote_file_path'])
else: #get
local_file_path = "%s/%s" %(django.conf.settings.DOWNLOAD_DIR,task_log_obj.task.id)
if not os.path.isdir(local_file_path):
os.mkdir(local_file_path)
file_name = task_data['remote_file_path'].split('/')[-1]
sftp.get(task_data['remote_file_path'], "%s/%s.%s" %(local_file_path,host.ip_addr,file_name))
task_log_obj.result = "get remote file [%s] succeeded" %(task_data['remote_file_path'])
t.close()
task_log_obj.status = 0
except Exception as e:
task_log_obj.result = e
task_log_obj.status = 1
task_log_obj.save()
if __name__ == '__main__':
base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
sys.path.append(base_dir)
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "CrazyEye.settings")
import django
django.setup()
from django import conf
from web import models
if len(sys.argv) == 1:
exit("error:must provide task_id!")
task_id = sys.argv[1]
task_obj = models.Task.objects.get(id=task_id)
#1. 生产多线程
pool = ThreadPoolExecutor(10)
if task_obj.task_type == 0:#cmd
thread_func = ssh_cmd
else: #file_transfer
thread_func = file_transfer
for task_log_detail_obj in task_obj.tasklogdetail_set.all():
pool.submit(thread_func,task_log_detail_obj)
#ssh_cmd(task_log_detail_obj)
pool.shutdown(wait=True)
2、CrazyEye
1、settings
1 import os 2 3 # Build paths inside the project like this: os.path.join(BASE_DIR, ...) 4 BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) 5 6 7 # Quick-start development settings - unsuitable for production 8 # See https://docs.djangoproject.com/en/1.10/howto/deployment/checklist/ 9 10 # SECURITY WARNING: keep the secret key used in production secret! 11 SECRET_KEY = 'c+lq-s!5j1($4zj+_3icw1xwr)yt#%x%&um#!e!b*-*5x(0&3a' 12 13 # SECURITY WARNING: don't run with debug turned on in production! 14 DEBUG = True 15 16 ALLOWED_HOSTS = ["*"] 17 18 19 # Application definition 20 21 INSTALLED_APPS = [ 22 'django.contrib.admin', 23 'django.contrib.auth', 24 'django.contrib.contenttypes', 25 'django.contrib.sessions', 26 'django.contrib.messages', 27 'django.contrib.staticfiles', 28 'web', 29 ] 30 31 MIDDLEWARE = [ 32 'django.middleware.security.SecurityMiddleware', 33 'django.contrib.sessions.middleware.SessionMiddleware', 34 'django.middleware.common.CommonMiddleware', 35 'django.middleware.csrf.CsrfViewMiddleware', 36 'django.contrib.auth.middleware.AuthenticationMiddleware', 37 'django.contrib.messages.middleware.MessageMiddleware', 38 'django.middleware.clickjacking.XFrameOptionsMiddleware', 39 ] 40 41 ROOT_URLCONF = 'CrazyEye.urls' 42 43 TEMPLATES = [ 44 { 45 'BACKEND': 'django.template.backends.django.DjangoTemplates', 46 'DIRS': [os.path.join(BASE_DIR, 'templates')] 47 , 48 'APP_DIRS': True, 49 'OPTIONS': { 50 'context_processors': [ 51 'django.template.context_processors.debug', 52 'django.template.context_processors.request', 53 'django.contrib.auth.context_processors.auth', 54 'django.contrib.messages.context_processors.messages', 55 ], 56 }, 57 }, 58 ] 59 60 WSGI_APPLICATION = 'CrazyEye.wsgi.application' 61 62 63 # Database 64 # https://docs.djangoproject.com/en/1.10/ref/settings/#databases 65 66 DATABASES = { 67 'default': { 68 'ENGINE': 'django.db.backends.sqlite3', 69 'NAME': os.path.join(BASE_DIR, 'db.sqlite3'), 70 } 71 } 72 73 74 # Password validation 75 # https://docs.djangoproject.com/en/1.10/ref/settings/#auth-password-validators 76 77 AUTH_PASSWORD_VALIDATORS = [ 78 { 79 'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator', 80 }, 81 { 82 'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator', 83 }, 84 { 85 'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator', 86 }, 87 { 88 'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator', 89 }, 90 ] 91 92 93 # Internationalization 94 # https://docs.djangoproject.com/en/1.10/topics/i18n/ 95 96 LANGUAGE_CODE = 'en-us' 97 98 TIME_ZONE = 'UTC' 99 100 USE_I18N = True 101 102 USE_L10N = True 103 104 USE_TZ = True 105 106 107 # Static files (CSS, JavaScript, Images) 108 # https://docs.djangoproject.com/en/1.10/howto/static-files/ 109 110 STATIC_URL = '/static/' 111 112 STATICFILES_DIRS = ( 113 os.path.join(BASE_DIR,'statics'), 114 ) 115 116 117 AUTH_USER_MODEL = 'web.UserProfile' 118 119 AUDIT_LOG_DIR = os.path.join(BASE_DIR,'log') 120 MULTITASK_SCRIPT= os.path.join(BASE_DIR,'backend/task_runner.py') 121 122 DOWNLOAD_DIR = os.path.join(BASE_DIR,'downloads') 123 124 125 LOGIN_URL = "/login/"