常用命令.md - 爱码网

#docker启动为知笔记 

sudo docker run --restart=on-failure:3 --name wiz -it -d -v /volume2/docker/wiz/data:/wiz/storage -v /etc/localtime:/etc/localtime -p 8888:80 -e SEARCH=true wiznote/wizserver 

sudo docker exec -it 775c7c9ee1e1 /bin/bash #进入容器 

docker logs -f -t --since="2017-05-31" --tail=10 edu_web_1 #--since : 此参数指定了输出日志开始日期，即只输出指定日期之后的日志。-f : 查看实时日志-t : 查看日志产生的日期-tail=10 : 查看最后的10条日志。edu_web_1 : 容器名称 

cd /var/lib/docker/containers/容器日志 

find -name '*zookeeper*' 

 ps -ef|grep zookeeper --color 

# 从主机复制到容器sudo docker cp host_path containerID:container_path 

#从容器复制到主机sudo docker cp containerID:container_path host_path 

#使用Dockerfile创建新的镜像(注意：末尾有一个点，表示当前路径):docker build -t "repository/images_name:tag" .    

docker commit 80daedc72971  docker.io/jiama/kylinchange:v2.1 

docker tag kylinchange docker.io/jiama/kylinchange:v2.1 

docker push docker.io/jiama/kylinchange:v2.1 

docker save -o xxx.tar.gz docker.io/jiama/kylinchange:v4.0 

#shell脚本 

#查找目录中的hadoop包含字符串内容那个的文件： 

for file in `hadoop fs -ls /user/epg/2020/1/11 |awk '{print $NF}'`; do echo $file; hadoop fs -text $file |grep "e_type':'open_advert.*p_type.*fa:16:3e:85:61:35" --color ; done 

hadoop fs -text /user/epg/2020/02/08/epg_log_2020020800????.log|grep "e_type':'order.*c8:1f:66:df:de:f0"|wc -l 

#hdfs清空回收站 

hdfs dfs -expunge  

#日志切割命令 

split -b 10m catalina.out 切割后的日志名前缀 

#批量删除后缀 

for i in `ls epglogs_send |grep epg-access-2019-08-13*`;do cp $i epglog20190914/${i%.*};done 

#存在的问题解决 

#cp: cannot stat ‘epg-access-2019-08-13-23-48.log.COMPLETED’: No such file or directory 

chown -R root:root gansulogs/ 

chmod -R 666 gansulogs/ 

mkdir epglog20190914 

#注意cp命令后的文件路径 

for i in `ls gansulogs | grep epg-access*`;do cp gansulogs/$i gansulogs/${i%.*};done 

#虚拟机 

#启动kafka 

nohup /opt/kafka_2.11-0.10.2.0/bin/kafka-server-start.sh /opt/kafka_2.11-0.10.2.0/config/server.properties > /opt/scriptlogs/kafka.log 2>&1 

#启动flume 

nohup /opt/flume/bin/flume-ng agent --conf conf --conf-file /opt/pukkasoft/flume/conf/nginx_kafka.properties --name myAgent -Dflume.root.logger=INFO,console > /opt/logs/flume.log 2>&1 

#创建消费者 

/opt/kafka_2.11-0.10.2.0/bin/kafka-console-consumer.sh --zookeeper localhost:2181 --topic ipvtepglogs 

#查看kafka leader 

/opt/kafka_2.11-0.10.2.0/bin/kafka-topics.sh --describe --topic report-request --zookeeper 192.168.20.160:2182,192.168.31.6:2181/apachekfk,192.168.31.7:2181/apachekfk 

#启动spark 

/opt/spark-1.6.3-bin-hadoop2.6/sbin/start-all.sh  

kafka-topics --delete --zookeeper 192.168.6.111:2181 --topic iptvepglogs #过一会主题就会被删掉 

连接hive 

 beeline -u jdbc:hive2://sh-dev3-demo:10000 -n hdfs 

#启动redis 

redis-server /opt/redis-3.2.8/redis.conf 

#CDH启动命令 

/opt/cloudera-manager/cm-5.16.1/etc/init.d/cloudera-scm-server start 

/opt/cloudera-manager/cm-5.16.1/etc/init.d/cloudera-scm-agent start 

service restart cloudera-scm-server									 

#cdh组件正在使用的配置 

/var/run/cloudera-scm-agent/process							 

#广信 

#广信回传ftp 

nohup sh -x /home/pake/scripts/log_ftp_2.sh 2019-08-16 2019-08-20 > /home/pake/logs/test.log 2>&1 & 

#mysql后台脚本执行脚本 

nohup mysql -h 192.168.10.45 -uroot -ppukkasoft2016 -Dintelligentrecommend < /home/pake/deleteWhere.sql > /home/pake/logs/delete.log 2>&1 & 

#ERROR 1205 (HY000) at line 1: Lock wait timeout exceeded; try restarting transaction 

#数据库操作最好分批进行特别是删除操作! 

delete from  intelligentrecommend.user_portrait_or_label_statistics where id>1400000 and id<'1600000' and content_kind_name ='精神'; 

创建数据库： 

CREATE DATABASE IF NOT EXISTS cmserver default character set utf8 COLLATE utf8_general_ci; 

 grant all privileges on sentry.* to 'sentryuser'@'%' identified by 'sentry321' with grant option; 

#根据标签查询推荐内容 

select t1.contentcode as  code,t1.contentname as name,'' as PosterImg,contenttype from t_cp_tag_content t1 where t1.TagName='文艺' and t1.ContentType in( 'program','series' ) and t1.ContentID not in (select t2.ContentID from t_cp_tag_content t2 where t2.TagName='不可推荐') limit 3; 

#快捷键 

#idea 

ctrl+r快速替换 

ctrl+shift+f全工程搜索 

#广信实时推荐job 

nohup spark-submit --name "报表统计" --class cn.pukkasoft.rec.job.CommonStatistics --master spark://192.168.31.5:7077 --deploy-mode cluster  hdfs://bigdata-cdh001.pukka.cn:8020/jobs/rec-statistics-job-20190908.jar jdbc:mysql://192.168.31.5:3306/ottdbsxxy?characterEncoding=utf8&useSSL=false root root hdfs://192.168.31.5:8020:/home/data/epg/2019/09/07/ 0931 > /opt/scriptlogs/testjob.log 2>&1 

#查看yarn程序日志 

yarn logs -applicationId  

测试日志写kafka 

#!/bin/bash 

i=1; 

while  

(( i <= $1 )) 

do 

        echo ["/log/s.gif?d={'c_content_id':1775291,'c_content_code':'cce07ad7a80a49a883f0579023c147ed','c_content_name':'精灵梦叶罗丽第 

七季 第1集','c_series_id':1775289,'c_series_code':'BC2C5B4000AF4D00FECF128A445D1654','c_series_name':'[HD]精灵梦叶罗丽 第七季','c_content_type':'series','c_sequence':1,'py_duration':84420,'py_times':0,'c_actors':'未知','c_writers':'未知','c_release_year':2016,'c_category_id':'1430,1439,6565,6595','c_category_name':'最新热播,全部少儿,7-12岁,魔法少女集结 闪耀吧仙子们','c_content_kind_id':'1629','c_content_kind_name':'少儿','c_cp_id':'11','c_cp_name':'MGTV_infuseCP','c_sp_id':11,'c_sp_name':'华为(青海)','o_id':'8001013020','o_name':'芒果包月 

包','c_is_pay':1,'u_id':'09768871181itv','u_group':'10','u_epg_group':'defaultnew','u_stbid':'00100516060100A000007847E31442DE','p_mark':'mango_play','e_type':'play','e_time':1567859185521,'p_action_time':1020087,'p_url':'http://125.72.108.136:8081/EPG720P/page/play.html?returnUrl=http%3A%2F%2F125.72.108.136%3A8081%2FEPG720P%2Fpage%2Fdetail-num.html%3FcontentId%3D1775289%26playNum%3D1%26recommendCode%3D1430','p_referrer':'http://125.72.108.136:8081/EPG720P/page/detail-num.html?contentId=1775289&contentType=3&recommendCode=1430','es_template_id':'0','es_site_name':'mango','p_type':'play','p_title':'芒果专区播放页','py_action':'heartbeat'}","07/Sep/2019:20:26:24 +0800","90:B1:1C:33:A0:44","192.168.11.27"]; 

       let i++; 

done | kafka-console-producer --broker-list bdp-reckon01-pukka:9092,bdp-reckon03-pukka:9092,bdp-reckon04-pukka:9092 --topic iptvepglogs 

sleep 10 

exit 0 

#tomcat启动关闭 

/opt/context/tomcat7094/bin/startup.sh  

/opt/context/tomcat7094/bin/shutdown.sh 

#nodejs 启动命令 

./restart.sh prod 

前臺運行npm run start 

#清空数据库 

##查出所有trunate语句 

select CONCAT('truncate TABLE ',table_schema,'.',TABLE_NAME, ';') from INFORMATION_SCHEMA.TABLES where  table_schema in ('iptvbigdata'); 

##复制查到的结果粘贴到查询中执行即可 

#杀死筛选进程号 

  kill `ps -ef|grep spark|awk '{print $2}'` 

#unzip命令解压缩jar包 

unzip xxx-web-role-1.0-SNAPSHOT.jar -d /APP/www/idoxu 

jar -cvfM0 project.jar ./ 

## kerbors 

##登录 

kinit -kt /run/cloudera-scm-agent/process/873-hdfs-NAMENODE-nnRpcWait/hdfs.keytab hdfs/bdp-reckon01-pukka@bdp-reckon01-pukka 

#presto启动所有节点 

/opt/cloudera/parcels/presto/bin/launcher start 

界面http://192.168.6.124:9999/ui/ 

#hivesql 

#增加列 

alter table dwd_play_fact_di add columns(`c_series_id` string COMMENT '剧头id',`c_series_name` string COMMENT '剧头名称',`c_series_code` string COMMENT '剧头code') 

#获取完整的oozie作业日志 

oozie job -log <job_id> 

#hive删除分区 

ALTER TABLE dwd_order_fact_di DROP IF EXISTS PARTITION (dt='2020-01-04');##测试没有成功删除

二、hql总结核心业务与关键字

累计: sum(xx) group by
topN： row_number() over(partition by xx order by yy desc) rank
a表存在b表不存在： left join on a.xx=b.xx and|where
join on and 不会过滤结果记录条数，只会根据and后的条件是否显示 B表的记录，A表的记录一定会显示

on and条件是在生成临时表时使用的条件，它不管on中的条件是否为真，都会返回左边表中的记录。
where条件是在临时表生成好后，再对临时表进行过滤的条件。这时已经没有left join的含义（必须返回左边表的记录）了，条件不为真的就全部过滤掉。

合并a表和b表数据: union [all] 所以加all更快

对重复结果的处理：UNION在进行表链接后会筛选掉重复的记录，Union All不会去除重复记录。
对排序的处理：Union将会按照字段的顺序进行排序；UNION ALL只是简单的将两个结果合并后就返回

行专列，列转行=>核心是把key的列枚举分类统计查询出value值的聚合结果

insert into TEST_TB_GRADE(USER_NAME, COURSE, SCORE)  values 

("张三", "数学", 34), 

("张三", "语文", 58), 

("张三", "英语", 58), 

("李四", "数学", 45), 

("李四", "语文", 87), 

("李四", "英语", 45), 

("王五", "数学", 76), 

("王五", "语文", 34), 

("王五", "英语", 89); 

---行专列 

SELECT user_name , 

    MAX(CASE course WHEN '数学' THEN score ELSE 0 END ) 数学, 

    MAX(CASE course WHEN '语文' THEN score ELSE 0 END ) 语文, 

    MAX(CASE course WHEN '英语' THEN score ELSE 0 END ) 英语 

FROM test_tb_grade 

GROUP BY USER_NAME; 

---列转行 

select user_name, '语文' COURSE , CN_SCORE as SCORE from test_tb_grade2 

union select user_name, '数学' COURSE, MATH_SCORE as SCORE from test_tb_grade2 

union select user_name, '英语' COURSE, EN_SCORE as SCORE from test_tb_grade2 

order by user_name,COURSE;