如何将数据转置到 hive

Question

这是我的蜂巢桌：

dept_id emp_cnt emp_cnt_prev_yr sales_cnt sales_cnt_prev_yr
1       10      8               10000     5000
2       15      9               20000     12000
3       12      10              30000     15000
4       6       12              40000     20000

我想将数据存储到另一个配置单元表中，如下所示：

dept_id metric_nm metric_val metric_val_prev-yr
1       emp_cnt   10         8
2       emp_cnt   15         9
3       emp_cnt   12         10
4       emp_cnt   6          12
1       sales_cnt 10000      5000
2       sales_cnt 20000      12000
3       sales_cnt 30000      15000
4       sales_cnt 40000      20000

到目前为止我尝试了什么：

 SELECT dept_id,
       metric_nm,
       Substr(metrics,1,Locate('#',metrics,1)-1) AS metric_val,
       Substr(metrics,Locate('#',metrics,1)+1)   AS metric_val_prev_yr
FROM   (
              SELECT dept_id,
                     Map('emp_cnt', Concat(emp_cnt,'#',emp_cnt_prev_yr),'sales_cnt', Concat(sales_cnt,'#',sales_cnt_prev_yr)) AS metrc
              FROM   <TABLE>) a lateral VIEW explode(metric) ext AS metric_nm,
       metrics; 

Answer 1

使用 UNION ALL 将两个 metric_nm 数据集合并为一个：

insert overwrite table table_name

select dept_id, 'emp_cnt' as metric_nm,  
       emp_cnt as metric_val, emp_cnt_prev_yr as metric_val_prev_yr
  from your_table

UNION ALL

select dept_id, 'sales_cnt' as metric_nm,  
       sales_cnt as metric_val, sales_cnt_prev_yr as metric_val_prev_yr
  from your_table;

另一种方法（与堆栈 metric_nm 交叉连接以将行 x metric_nm 值的数量相乘），此 CROSS JOIN 将作为 map-join 执行：

--configuration
set hive.cli.print.header=true;
set hive.execution.engine=tez;
set hive.mapred.reduce.tasks.speculative.execution=false;
set mapred.reduce.tasks.speculative.execution=false;
set hive.exec.parallel=true;
set hive.exec.parallel.thread.number=36;
set hive.vectorized.execution.enabled=true;
set hive.vectorized.execution.reduce.enabled=true;
set hive.auto.convert.join=true; --this enables map-join

select dept_id, s.metric_nm,  

           case s.metric_nm when 'emp_cnt'   then emp_cnt 
                            when 'sales_cnt' then sales_cnt 
                            --add more cases
            end as metric_val, 

           case s.metric_nm when 'emp_cnt'   then emp_cnt_prev_yr 
                            when 'sales_cnt' then sales_cnt_prev_yr 
                            --add more cases
            end as metric_val_prev_yr

      from your_table 
           cross join 
           (select stack (2, --number of values, add more 
                          'sales_cnt', 
                          'emp_cnt'
                          --add more values
                         ) as metric_nm
           )s