ES全量索引校验-java

import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.qa.utils.*;
import org.testng.annotations.DataProvider;
import org.testng.annotations.Test;
 
import java.io.BufferedReader;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.sql.*;
import java.util.*;
 
public class GBIndexCheck3 {
 
    @DataProvider(name = "mysqlData")
    public static Object[][] toBeDataProvider() throws SQLException, ClassNotFoundException {
        return getTestData();
    }
 
    @Test(dataProvider = "mysqlData")
    //@Test(dataProvider = "testData")
    public void compareIndex(String goodsSn, String whcode) throws Exception {
        //请求索引一
        String goodsId = goodsSn + "#" + whcode;
        String esURL = "http://xxxx/Full_Index_test_GB_old/_search";
        String esSheetName = "es";
        List<String> list1 = indexGB(goodsId, esURL, esSheetName);
 
        //请求索引二
        String aiURL = "http://xxxx/GB_daily_full/_search";
        String aiSheetName = "ai";
        List<String> list2 = indexGB(goodsId, aiURL, aiSheetName);
 
        //判断两份索引数据的一致性
        Boolean falg = false;
        falg = isListEqual(list1, list2);
        if (!falg) {
            //System.out.println("【索引数据不一致】");
            //索引数据不一致sku写入txt
            WriteTxtUtil.writerForString(goodsId);
        }
    }
 
 
    public List indexGB(String goodsId, String url, String sheetName) throws Exception {
 
        //获取索引路径
        //String url = (String) PropertiesUtil.getProperties().get("GB.es.url");
        //String url = "http://10.4.4.80:9200/gearbest20191205141505/_search";
        //获取参数
        //String params = (String) PropertiesUtil.getProperties().get("GB.es.params");
        String params = "{\"query\":{\"term\":{\"goodsId\":{\"value\":\"" + goodsId + "\"}}}}";
        //String params = "{\"query\":{\"term\":{\"goodsSn\":{\"value\":\"244297701\"}}}}";
 
        System.out.println(params);
        //响应结果
        JSONObject responseObject = RestClient.post(url, params);
        System.out.println(sheetName + " ---> " + "responce: " + responseObject);
 
        JSONObject data = (JSONObject) responseObject.get("hits");
        JSONArray jsonArray = data.getJSONArray("hits");
 
        //遍历jsonArray
        Map<String, Object> map = new HashMap<>(); //索引字段数据存map
        List<String> list = new ArrayList<>(); //MD5加密后数据存list
 
        for (int i = 0; i < jsonArray.size(); i++) {
            JSONObject object = (JSONObject) jsonArray.get(i);
            JSONObject source = (JSONObject) object.get("_source");
            //内嵌对象排序
            JSONArray skuDescAttrs = source.getJSONArray("skuDescAttrs");
            skuDescAttrs.sort(Comparator.comparing(e -> ((JSONObject) e).getInteger("attrValueKey")));
            map.put("skuDescAttrs", skuDescAttrs.toString());
            //
            map.put("week2SalesVolume", source.getString("week2SalesVolume"));
            map.put("payEndTime", source.getString("payEndTime"));
            map.put("appSwellAmount", source.getString("appSwellAmount"));
            map.put("appPriceType", source.getString("appPriceType"));
            map.put("goodsId", source.getString("goodsId"));
            map.put("youtube", source.getString("youtube"));
 
            //skuAttrs
            JSONArray skuAttrs = source.getJSONArray("skuAttrs");
            skuAttrs.sort(Comparator.comparing(e->((JSONObject)e).getInteger("attrType")));
            map.put("skuAttrs",skuAttrs.toString());
 
            map.put("week2Sales", source.getString("week2Sales"));
            map.put("shopPrice", source.getString("shopPrice"));
            //grossMargin 先写死
            //map.put("grossMargin",source.getString("grossMargin"));
            map.put("baseScore2", source.getString("baseScore2"));
            map.put("subTitle", source.getString("subTitle"));
            map.put("exposureFlag", source.getString("exposureFlag"));
            map.put("originalUrl", source.getString("originalUrl"));
            map.put("discount", source.getString("discount"));
            map.put("whCode", source.getString("whCode"));
            map.put("baseScore5", source.getString("baseScore5"));
            map.put("payStartTime", source.getString("payStartTime"));
            map.put("recommendedLevel", source.getString("recommendedLevel"));
            map.put("thumbExtendUrl", source.getString("thumbExtendUrl"));
            map.put("appStatus", source.getString("appStatus"));
            map.put("passAvgScore", source.getString("passAvgScore"));
            map.put("stockFlag", source.getString("stockFlag"));
            map.put("brandName", source.getString("brandName"));
            map.put("appDisplayPrice", source.getString("appDisplayPrice"));
            map.put("centerWord", source.getString("centerWord"));
            map.put("goodsTitle", source.getString("goodsTitle"));
            map.put("brandCode", source.getString("brandCode"));
            map.put("appDeposit", source.getString("appDeposit"));
            map.put("vWhCode", source.getString("vWhCode"));
            map.put("catId", source.getString("catId"));
            map.put("priceRates", source.getString("priceRates"));
            map.put("totalFavoriteCount", source.getString("totalFavoriteCount"));
            map.put("baseScore4", source.getString("baseScore4"));
            map.put("appPayEndTime", source.getString("appPayEndTime"));
            map.put("urlTitle", source.getString("urlTitle"));
 
            //categories
            JSONArray categories = source.getJSONArray("categories");
            categories.sort(Comparator.comparing(e->((JSONObject)e).getInteger("catId")));
            map.put("categories",categories.toString());
 
            map.put("firstUpTime", source.getString("firstUpTime"));
            // labelFlags
            JSONArray labelFlags = source.getJSONArray("labelFlags");
            labelFlags.sort(Comparator.comparing(e->((JSONObject)e).getInteger("labelId")));
            map.put("labelFlags",labelFlags.toString());
 
 
            map.put("appPayStartTime", source.getString("appPayStartTime"));
            map.put("expiredTime", source.getString("expiredTime"));
            map.put("exposureSalesVolume", source.getString("exposureSalesVolume"));
            map.put("appDiscount", source.getString("appDiscount"));
            map.put("baseScore1", source.getString("baseScore1"));
            //
            JSONArray coupons = source.getJSONArray("coupons");
            coupons.sort(Comparator.comparing(e->((JSONObject)e).getInteger("code")));
            map.put("coupons",coupons.toString());
 
            map.put("lang", source.getString("lang"));
            map.put("isCod", source.getString("isCod"));
            map.put("searchWords", source.getString("searchWords"));
            //defaultWh 先写死
            //map.put("defaultWh",source.getString("defaultWh"));
            map.put("mStatus", source.getString("mStatus"));
            map.put("dailyRate", source.getString("dailyRate"));
            map.put("exposureSalesRate", source.getString("exposureSalesRate"));
            map.put("sortOrder", source.getString("sortOrder"));
            map.put("shopGroups", source.getString("shopGroups"));
            map.put("passTotalNum", source.getString("passTotalNum"));
            map.put("shopCode", source.getString("shopCode"));
            map.put("appExpiredTime", source.getString("appExpiredTime"));
            map.put("isTort", source.getString("isTort"));
            map.put("baseScore3", source.getString("baseScore3"));
            map.put("priceType", source.getString("priceType"));
            map.put("createTime", source.getString("createTime"));
            map.put("swellAmount", source.getString("swellAmount"));
            map.put("saleMark", source.getString("saleMark"));
            map.put("deposit", source.getString("deposit"));
            map.put("goodsModelWord", source.getString("goodsModelWord"));
            //appDefaultWh 先写死
            //map.put("appDefaultWh",source.getString("appDefaultWh"));
            map.put("goodsWebSku", source.getString("goodsWebSku"));
            map.put("imgExtendUrl", source.getString("imgExtendUrl"));
            //activityIds 数组
            //map.put("activityIds",source.getString("activityIds"));
 
            map.put("displayPrice", source.getString("displayPrice"));
            map.put("goodsWebSpu", source.getString("goodsWebSpu"));
            map.put("isPlatform", source.getString("isPlatform"));
            map.put("webStatus", source.getString("webStatus"));
            map.put("goodsSn", source.getString("goodsSn"));
            map.put("imgUrl", source.getString("imgUrl"));
            // tags 数组
            //map.put("tags",source.getString("tags"));
            //map.put("yesterdaySales",source.getString("yesterdaySales"));
            map.put("yesterdaySales", 0);
 
            JSONArray activities = source.getJSONArray("activities");
            activities.sort(Comparator.comparing(e->((JSONObject)e).getInteger("activityId")));
            map.put("activities",activities.toString());
 
            //遍历map
            for (Map.Entry<String, Object> entry : map.entrySet()) {
                //Log.info("索引方案"+sheetName);
                Log.info(entry.getKey() + ":" + entry.getValue());
            }
 
            // MD5处理
            String str = MapToString.getMapToString(map);
            System.out.println("map转string: " + str);
            String str2 = goodsId + "-" + MD5Util.md5(str);
            System.out.println("MD5加密后：" + str2);
            list.add(str2);
        }
        //ESWriteExcelUtilsForList.doWrite(list,"D://索引测试数据.xlsx",sheetName,0,1);
        return list;
    }
 
    // 内嵌对象排序
    public void  arraySort(JSONArray jsonArray,String...values){
        jsonArray.sort((a,b)->{
            int i = ((JSONObject)a).getString(values[0]).compareTo(((JSONObject)b).getString(values[0]));
                if(i==0){
                    int j = ((JSONObject)a).getInteger(values[1]).compareTo(((JSONObject)b).getInteger(values[1]));
                    return j;
                }
            return i;
        });
    }
 
 
    //获取Mysql数据库中的测试数据
    public static Object[][] getTestData() throws ClassNotFoundException, SQLException {
        //设置mysql驱动
        Class.forName("com.mysql.jdbc.Driver");
        //建立数据库的连接    10.60.46.206:3307
        Connection conn = DriverManager.getConnection("jdbc:mysql://10.40.2.131:3306/gb_goods", "java-service", "java123456");
        //Connection conn = DriverManager.getConnection("jdbc:mysql:///test", "root", "root");
        //判断数据连接是否成功
        if (!conn.isClosed()) {
            System.out.println("连接成功");
        } else {
            System.out.println("连接失败");
        }
        //创建Statement对象可以用对应的方法executeQuery(sql语句)获取测试数据
        Statement statement = conn.createStatement();
        //创建一个结果集存放数据库执行完sql的数据
        //select DISTINCT good_sn from goods_info where good_sn in('237646510','258050019')
 
        String sql = "SELECT gie.good_sn,gie.v_wh_code AS goodsSn FROM goods_info_extend_s_53   gie WHERE gie.goods_status IN (2,4,5) AND gie.platform IN (1,2,4) AND gie.site_code = 'GB' GROUP BY gie.good_sn, gie.v_wh_code;";
        //String sql ="SELECT goods_sn,1 from kp_goods;";
        ResultSet resultSet = statement.executeQuery(sql);
        //声明存放泛型string数组的list对象
        List<Object[]> list = new ArrayList<Object[]>();
        int colNum = resultSet.getMetaData().getColumnCount();
        while (resultSet.next()) {
            String[] strings = new String[colNum];
            for (int i = 0; i < strings.length; i++) {
                strings[i] = resultSet.getString(i + 1);
            }
            list.add(strings);
        }
        resultSet.close();
        conn.close();
 
        //将list对象转换成二维数组
        Object[][] results = new Object[list.size()][];
        for (int i = 0; i < list.size(); i++) {
            results[i] = list.get(i);
        }
        return results;
    }
 
 
    //比较两个list内容是否一致
    public static boolean isListEqual(List l0, List l1) {
        if (l0 == l1)
            return true;
        if (l0 == null && l1 == null)
            return true;
        if (l0 == null || l1 == null)
            return false;
        if (l0.size() != l1.size())
            return false;
        for (Object o : l0) {
            if (!l1.contains(o))
                return false;
        }
        for (Object o : l1) {
            if (!l0.contains(o))
                return false;
        }
        return true;
    }
 
 
    //csv dataProvider
    @DataProvider(name = "testData")
    public static Object[][] words() throws IOException {
        return getTestData1("D://kp_goodsSn.csv");
    }
 
    //读取CSV文件的方法
    public static Object[][] getTestData1(String fileName) throws IOException {
        //定义一个集合，存csv文件中的数据
        List<Object[]> records = new ArrayList<Object[]>();
        String record;
        BufferedReader file = new BufferedReader(new InputStreamReader(new FileInputStream(fileName), "UTF-8"));
        //file.readLine(); //跳过表头, 如果不需要表头的话，不要写这句
        while ((record = file.readLine()) != null) {
            String fields[] = record.split(",");
            records.add(fields);
        }
        file.close();
 
        //定义方法的返回值，将list转换为Object二维数据
        Object[][] results = new Object[records.size()][];
        //设置二维数每行的值，每行是一个Object对象
        for (int i = 0; i < records.size(); i++) {
            results[i] = records.get(i);
        }
        return results;
    }
}

import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import org.apache.http.HttpResponse;
import org.apache.http.HttpStatus;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.entity.StringEntity;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;
import java.nio.charset.Charset;
 
public class RestClient {
 
    /**
     * 带json参数的post请求
     * @param url
     * @param parms
     * @return
     */
    public static JSONObject post(String url, String parms) {
 
        CloseableHttpClient httpClient = HttpClients.createDefault();
        HttpPost post = new HttpPost(url);
 
        post.addHeader("content-type", "application/json;charset=utf-8");
        post.setEntity(new StringEntity(parms, Charset.forName("utf-8")));
        HttpResponse response = null;
        String result = null;
        try {
            Log.info("开始发送post请求，请求的URL: " + url);
            Log.info("开始发送post请求，请求的参数: " + parms);
            response = httpClient.execute(post);
            result = EntityUtils.toString(response.getEntity());
        } catch (Exception e) {
            e.printStackTrace();
        }
        if (HttpStatus.SC_OK != response.getStatusLine().getStatusCode()) {
            System.out.println("请求不正确");
        }
 
        JSONObject responseObject = JSON.parseObject(result);
        //System.out.println("response: "+responseObject);
        return responseObject;
        //String data = jsonObject.getString("hits");
        /*JSONObject data = (JSONObject) jsonObject.get("hits");
        //System.out.println("data--->: "+data);
        JSONArray jsonArray = data.getJSONArray("hits");
 
        Map<String,Object> hmap = new HashMap<>();
        for (int i=0;i<jsonArray.size();i++){
            JSONObject object = (JSONObject) jsonArray.get(i);
            //System.out.println("_source: " + object.get("_source") +"," + "values: " + object.get("_source"));
            JSONObject str = (JSONObject) object.get("_source");
            System.out.println("str:"+str);
           // System.out.println("productSource:"+str.get("productSource"));
*/
    }
}

import org.testng.annotations.Test;
import java.security.MessageDigest;
 
public class MD5Util {
    /**
     * MD5加密
     * @param string
     * @return
     */
    public static String md5(String string) {
        try {
            // 得到一个信息摘要器
            MessageDigest digest = MessageDigest.getInstance("md5");
            byte[] result = digest.digest(string.getBytes());
            StringBuffer buffer = new StringBuffer();
            // 把每一个byte 做一个与运算 0xff;
            for (byte b : result) {
                // 与运算
                int number = b & 0xff;// 加盐
                String str = Integer.toHexString(number);
                if (str.length() == 1) {
                    buffer.append("0");
                }
                buffer.append(str);
            }
 
            // 标准的md5加密后的结果
            return buffer.toString();
        } catch (Exception e) {
            e.printStackTrace();
            return "";
        }
    }
 
 
    @Test
    public void test(){
        String str = "abcde123";
        System.out.println("加密前："+str);
        System.out.println("加密后："+md5(str));
    }
 
}

import java.io.FileWriter;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
 
public class WriteTxtUtil {
 
    public static void writeForMap(Map<String,Object> map) throws Exception{
        try {
            String line = System.getProperty("line.separator");
            StringBuffer str = new StringBuffer();
            FileWriter writer = new FileWriter("D:\\sms_cloud.txt",true);
            Set set = map.entrySet();
            Iterator in = set.iterator();
            while (in.hasNext()){
                Map.Entry<String,Object> entry = (Map.Entry<String, Object>) in.next();
                str.append(entry.getKey()+" = "+entry.getValue()).append(line);
            }
            writer.write(str.toString());
            System.out.println("文件写入完毕");
            writer.close();
        } catch (Exception e){
            e.printStackTrace();
        }
    }
 
    public static void writerForList(List<String> list) throws Exception{
        try {
            String line = System.getProperty("line.separator");
            StringBuffer sb = new StringBuffer();
            FileWriter writer = new FileWriter("D:\\test.txt",true);
            for(int i = 0;i < list.size(); i ++){
                //System.out.println(list.get(i));
                sb.append("goodsSn:"+list.get(i)).append(line);
            }
            writer.write(sb.toString());
            System.out.println("数据写入完毕");
            writer.close();
        } catch (Exception e){
            e.printStackTrace();
        }
    }
 
    public static void writerForString(String string) throws Exception{
        try {
            String line = System.getProperty("line.separator");
            FileWriter writer = new FileWriter("D:\\索引数据不一致sku.txt",true);
            writer.write(string+"\n");
            System.out.println("数据写入成功");
            writer.close();
        }catch (Exception e){
            e.printStackTrace();
        }
    }
}

import com.qa.utils.ReadProperties;
 
import java.util.Properties;
 
public class PropertiesUtil {
 
    public static Properties getProperties(){
 
        Properties properties = ReadProperties.ReadProperties();
        return properties;
    }
}

ZF.ZF_en.filePath=F://索引测试//ZF_en索引.xlsx
ZF.ZF_en.sheetName=ZF_en
ZF.ZF_en.sql=；
ZF.ZF_en.es.url=http://xxxx/ZF_en/_search
ZF.ZF_en.es.params={"query":{"term":{"goodsSn":{"value":"205787502"}}}}
ZF.ZF_en.api.url=http://xxxx/zf/search
ZF.ZF_en.api.params={"domain":"ZF_en","accessToken":"dc698883cbca47bbac191787969f9de4","version":5,"cache":false,"agent":"web","pageSize":120,"logEnable":true,"pageNo":1,"filters":[{"field":"goodsSn","values":["205787502"]}]}