【发布时间】:2020-09-02 22:51:04
【问题描述】:
我正在处理我们的报告数据库。我们当前的数据转储过程遇到了很多问题,我正在尝试对其进行修改。
每个小时,我们都会将数据从生产中添加到我们的表中。具体来说,它有超过一百万行(通常更少)。我需要抓取这些行并将其附加到我们的报告服务器表中。目前,我正在使用一个使用 MERGE 过程的存储过程(使用目标、源技术)。运行大约需要每小时 16 分钟。任何人都有更好的想法来追加/更新行?我们点击的生产表也只包含 TODAY's 数据,所以我什至不需要放入日期过滤器来减少数据负载。优化性能的最佳方法是什么? 这里是执行计划:https://www.brentozar.com/pastetheplan/?id=HJrfFsTXP 流程如下:
CREATE PROCEDURE dbo.spINSERT_FULL_ORDERHEADER
AS
BEGIN
WITH ORDER_DETAIL_V2 AS(
SELECT
'DCMS' AS SOURCE,
DC_ORDER_NUMBER,
DCMS_ORDER_TYPE,
CUSTOMER_PURCHASE_ORDER_ID,
BILL_TO,
CUSTOMER_MASTER_RECORD_TYPE,
SHIP_TO,
SALES_ORDER,
MERCHANDISE_STYLE,
MERCHANDISE_SIZE_1,
CONCAT(SUBSTRING(MERCHANDISE_STYLE,3,6),'-',
SUBSTRING(MERCHANDISE_STYLE,9,3)) AS MATERIAL,
SUBSTRING(MERCHANDISE_STYLE,14,2) AS QUALITY,
ORDER_QTY,
ORDER_SELECTED_QTY,
ORDER_SHIPPED_QTY,
RELEASED_QTY,
PICK_QTY,
PACKED_QTY,
RELEASED_CARTONS,
FULL_CASE_PICK_COUNT,
TOTE_COUNT,
SPECIAL_PROCESS_CODE_1,
SPECIAL_PROCESS_CODE_2,
SPECIAL_PROCESS_CODE_3,
ORDER_DETAIL_STATUS,
ORDER_CARRIER,
CARRIER_SERVICE_ID,
CAST(CONVERT(VARCHAR, CREATE_DATE) AS DATE) AS CREATE_DATE,
LEFT(cast(dateadd(hh,2,LEFT(RIGHT(CONCAT('000000', CREATE_TIME),6), 2)+':'+SUBSTRING(RIGHT(CONCAT('000000', CREATE_TIME), 6), 3, 2)+':'+RIGHT(RIGHT(CONCAT('000000', CREATE_TIME), 8), 2)) AS TIME),8) AS CREATE_TIME,
CAST(CONVERT(VARCHAR, ALLOCATION_DATE) AS DATE) AS ALLOCATION_DATE,
CAST(CONVERT(VARCHAR, REQUESTED_SHIP_DATE) AS DATE) AS REQUESTED_SHIP_DATE,
CAST(CONVERT(VARCHAR, CANCEL_DATE) AS DATE) AS CANCEL_DATE,
CASE
WHEN DISPATCH_DATE = 0
THEN NULL
ELSE CAST(CONVERT(VARCHAR, DISPATCH_DATE) AS DATE)
END AS DISPATCH_DATE,
CASE
WHEN RELEASED_DATE = 0
THEN NULL
ELSE CAST(CONVERT(VARCHAR, RELEASED_DATE) AS DATE)
END AS RELEASED_DATE,
LEFT(RIGHT(CONCAT('000000', RELEASED_TIME), 6), 2)+':'+SUBSTRING(RIGHT(CONCAT('000000',
RELEASED_TIME), 6), 3, 2) +':'+ RIGHT(RIGHT(CONCAT('000000', RELEASED_TIME), 6), 2) AS RELEASED_TIME,
PRIORITY_ORDER,
PARTIAL_SHIP,
SHIPMENT_CARTONS,
SCANNED_CARTONS,
STAGED_CARTONS,
LOADED_CARTONS,
CASE
WHEN INVOICE_DATE = 0
THEN NULL
ELSE CAST(CONVERT(VARCHAR, INVOICE_DATE) AS DATE)
END AS INVOICE_DATE,
SHIPPING_LOAD_NUMBER,
STARTING_WAVE,
ENDING_WAVE,
CASE
WHEN STAGED_DATE = 0
THEN NULL
ELSE CAST(CONVERT(VARCHAR, STAGED_DATE) AS DATE)
END AS STAGED_DATE,
ORDER_HDR_STATUS,
LEFT(HOST_ORDER_NUMBER, 9) AS DELIVERY_NUMBER,
CASE
WHEN NOT_SELECTED_DATE = 0
THEN NULL
ELSE CAST(CONVERT(VARCHAR, NOT_SELECTED_DATE) AS DATE)
END AS NOT_SELECTED_DATE,
LEFT(RIGHT(CONCAT('000000', NOT_SELECTED_TIME), 6), 2)+':'+SUBSTRING(RIGHT(CONCAT('000000', NOT_SELECTED_TIME), 6), 3, 2)+':'+RIGHT(RIGHT(CONCAT('000000', NOT_SELECTED_TIME), 6), 2) AS NOT_SELECTED_TIME,
NOT_SELECTED_REASON,
CAST(CONVERT(VARCHAR, EST_DEL_DT) AS DATE) AS EST_DEL_DATE,
CONCAT(DC_ORDER_NUMBER,MERCHANDISE_STYLE,MERCHANDISE_SIZE_1,CREATE_DATE) AS [KEY]
FROM OPENQUERY(MEMDWPR1,'
SELECT
OHORNO AS DC_ORDER_NUMBER,
OHORTY AS DCMS_ORDER_TYPE,
OHCUPO AS CUSTOMER_PURCHASE_ORDER_ID,
OHBLTO AS BILL_TO,
OHCURT AS CUSTOMER_MASTER_RECORD_TYPE,
OHSHTO AS SHIP_TO,
OHSAOR AS SALES_ORDER,
ODMRSY AS MERCHANDISE_STYLE,
ODMRS1 AS MERCHANDISE_SIZE_1,
ODORQT AS ORDER_QTY,
ODORSL AS ORDER_SELECTED_QTY,
ODORSH AS ORDER_SHIPPED_QTY,
OHRLQT AS RELEASED_QTY,
OHPKQT AS PICK_QTY,
OHPDQT AS PACKED_QTY,
OHRLCA AS RELEASED_CARTONS,
OHFCCT AS FULL_CASE_PICK_COUNT,
OHTOCT AS TOTE_COUNT,
ODSP01 AS SPECIAL_PROCESS_CODE_1,
ODSP02 AS SPECIAL_PROCESS_CODE_2,
ODSP03 AS SPECIAL_PROCESS_CODE_3,
ODODST AS ORDER_DETAIL_STATUS,
OHORCR AS ORDER_CARRIER,
OHCRSV AS CARRIER_SERVICE_ID,
OHCRDT AS CREATE_DATE,
OHCRTM AS CREATE_TIME,
OHALDT AS ALLOCATION_DATE,
OHRQDT AS REQUESTED_SHIP_DATE,
OHCNDT AS CANCEL_DATE,
OHDIDT AS DISPATCH_DATE,
OHRLDT AS RELEASED_DATE,
OHRLTM AS RELEASED_TIME,
OHORPY AS PRIORITY_ORDER,
OHPLSH AS PARTIAL_SHIP,
OHSHCA AS SHIPMENT_CARTONS,
OHSNCA AS SCANNED_CARTONS,
OHSGCA AS STAGED_CARTONS,
OHLDCA AS LOADED_CARTONS,
OHIVDT AS INVOICE_DATE,
OHLDNO AS SHIPPING_LOAD_NUMBER,
OHWVST AS STARTING_WAVE,
OHWVED AS ENDING_WAVE,
OHSGDT AS STAGED_DATE,
OHOHST AS ORDER_HDR_STATUS,
OHHSOR AS HOST_ORDER_NUMBER,
OHSLDT AS NOT_SELECTED_DATE,
OHSLTM AS NOT_SELECTED_TIME,
OHSLRS AS NOT_SELECTED_REASON,
OHEXDD AS EST_DEL_DT
FROM DSV052BDTA.OHORDR00@MEMWMS2.WORLD
JOIN DSV052BDTA.ODORDR00@MEMWMS2.WORLD
ON ODORNO = OHORNO
')
GROUP BY
DC_ORDER_NUMBER,
DCMS_ORDER_TYPE,
CUSTOMER_PURCHASE_ORDER_ID,
BILL_TO,
CUSTOMER_MASTER_RECORD_TYPE,
SHIP_TO,
SALES_ORDER,
MERCHANDISE_STYLE,
MERCHANDISE_SIZE_1,
CONCAT(SUBSTRING(MERCHANDISE_STYLE,3,6),'-',
SUBSTRING(MERCHANDISE_STYLE,9,3)),
SUBSTRING(MERCHANDISE_STYLE,14,2),
ORDER_QTY,
ORDER_SELECTED_QTY,
ORDER_SHIPPED_QTY,
RELEASED_QTY,
PICK_QTY,
PACKED_QTY,
RELEASED_CARTONS,
FULL_CASE_PICK_COUNT,
TOTE_COUNT,
SPECIAL_PROCESS_CODE_1,
SPECIAL_PROCESS_CODE_2,
SPECIAL_PROCESS_CODE_3,
ORDER_DETAIL_STATUS,
ORDER_CARRIER,
CARRIER_SERVICE_ID,
CAST(CONVERT(VARCHAR, CREATE_DATE) AS DATE),
LEFT(cast(dateadd(hh,2,LEFT(RIGHT(CONCAT('000000', CREATE_TIME),6), 2)+':'+SUBSTRING(RIGHT(CONCAT('000000', CREATE_TIME), 6), 3, 2)+':'+RIGHT(RIGHT(CONCAT('000000', CREATE_TIME), 8), 2)) AS TIME),8),
CAST(CONVERT(VARCHAR, ALLOCATION_DATE) AS DATE) ,
CAST(CONVERT(VARCHAR, REQUESTED_SHIP_DATE) AS DATE) ,
CAST(CONVERT(VARCHAR, CANCEL_DATE) AS DATE) ,
CASE
WHEN DISPATCH_DATE = 0
THEN NULL
ELSE CAST(CONVERT(VARCHAR, DISPATCH_DATE) AS DATE)
END,
CASE
WHEN RELEASED_DATE = 0
THEN NULL
ELSE CAST(CONVERT(VARCHAR, RELEASED_DATE) AS DATE)
END,
LEFT(RIGHT(CONCAT('000000', RELEASED_TIME), 6), 2)+':'+SUBSTRING(RIGHT(CONCAT('000000',
RELEASED_TIME), 6), 3, 2) +':'+ RIGHT(RIGHT(CONCAT('000000', RELEASED_TIME), 6), 2) ,
PRIORITY_ORDER,
PARTIAL_SHIP,
SHIPMENT_CARTONS,
SCANNED_CARTONS,
STAGED_CARTONS,
LOADED_CARTONS,
CASE
WHEN INVOICE_DATE = 0
THEN NULL
ELSE CAST(CONVERT(VARCHAR, INVOICE_DATE) AS DATE)
END,
SHIPPING_LOAD_NUMBER,
STARTING_WAVE,
ENDING_WAVE,
CASE
WHEN STAGED_DATE = 0
THEN NULL
ELSE CAST(CONVERT(VARCHAR, STAGED_DATE) AS DATE)
END,
ORDER_HDR_STATUS,
LEFT(HOST_ORDER_NUMBER, 9),
CASE
WHEN NOT_SELECTED_DATE = 0
THEN NULL
ELSE CAST(CONVERT(VARCHAR, NOT_SELECTED_DATE) AS DATE)
END,
LEFT(RIGHT(CONCAT('000000', NOT_SELECTED_TIME), 6), 2)+':'+SUBSTRING(RIGHT(CONCAT('000000', NOT_SELECTED_TIME), 6), 3, 2)+':'+RIGHT(RIGHT(CONCAT('000000', NOT_SELECTED_TIME), 6), 2),
NOT_SELECTED_REASON,
CAST(CONVERT(VARCHAR, EST_DEL_DT) AS DATE),
CONCAT(DC_ORDER_NUMBER,MERCHANDISE_STYLE,MERCHANDISE_SIZE_1,CREATE_DATE)
)
MERGE BUSINESSPLANNING_TEST.dbo.[FullOrderHeader] AS TARGET
USING ORDER_DETAIL_V2 AS SOURCE
ON TARGET.[KEY] = SOURCE.[KEY]
WHEN MATCHED THEN
UPDATE SET
TARGET.[SOURCE] = SOURCE.[SOURCE],
TARGET.DC_ORDER_NUMBER = SOURCE.DC_ORDER_NUMBER,
TARGET.DCMS_ORDER_TYPE = SOURCE.DCMS_ORDER_TYPE,
TARGET.CUSTOMER_PURCHASE_ORDER_ID = SOURCE.CUSTOMER_PURCHASE_ORDER_ID,
TARGET.BILL_TO = SOURCE.BILL_TO,
TARGET.CUSTOMER_MASTER_RECORD_TYPE = SOURCE.CUSTOMER_MASTER_RECORD_TYPE,
TARGET.SHIP_TO = SOURCE.SHIP_TO,
TARGET.SALES_ORDER = SOURCE.SALES_ORDER,
TARGET.MERCHANDISE_STYLE = SOURCE.MERCHANDISE_STYLE,
TARGET.MERCHANDISE_SIZE_1 = SOURCE.MERCHANDISE_SIZE_1,
TARGET.MATERIAL = SOURCE.MATERIAL,
TARGET.QUALITY = SOURCE.QUALITY,
TARGET.ORDER_QTY = SOURCE.ORDER_QTY,
TARGET.ORDER_SELECTED_QTY = SOURCE.ORDER_SELECTED_QTY,
TARGET.ORDER_SHIPPED_QTY = SOURCE.ORDER_SHIPPED_QTY,
TARGET.RELEASED_QTY = SOURCE.RELEASED_QTY,
TARGET.PICK_QTY = SOURCE.PICK_QTY,
TARGET.PACKED_QTY = SOURCE.PACKED_QTY,
TARGET.RELEASED_CARTONS = SOURCE.RELEASED_CARTONS,
TARGET.FULL_CASE_PICK_COUNT = SOURCE.FULL_CASE_PICK_COUNT,
TARGET.TOTE_COUNT = SOURCE.TOTE_COUNT,
TARGET.SPECIAL_PROCESS_CODE_1 = SOURCE.SPECIAL_PROCESS_CODE_1,
TARGET.SPECIAL_PROCESS_CODE_2 = SOURCE.SPECIAL_PROCESS_CODE_2,
TARGET.SPECIAL_PROCESS_CODE_3 = SOURCE.SPECIAL_PROCESS_CODE_3,
TARGET.ORDER_DETAIL_STATUS = SOURCE.ORDER_DETAIL_STATUS,
TARGET.ORDER_CARRIER = SOURCE.ORDER_CARRIER,
TARGET.CARRIER_SERVICE_ID = SOURCE.CARRIER_SERVICE_ID,
TARGET.CREATE_DATE = SOURCE.CREATE_DATE,
TARGET.CREATE_TIME = SOURCE.CREATE_TIME,
TARGET.ALLOCATION_DATE = SOURCE.ALLOCATION_DATE,
TARGET.REQUESTED_SHIP_DATE = SOURCE.REQUESTED_SHIP_DATE,
TARGET.CANCEL_DATE = SOURCE.CANCEL_DATE,
TARGET.DISPATCH_DATE = SOURCE.DISPATCH_DATE,
TARGET.RELEASED_DATE = SOURCE.RELEASED_DATE,
TARGET.RELEASED_TIME = SOURCE.RELEASED_TIME,
TARGET.PRIORITY_ORDER = SOURCE.PRIORITY_ORDER,
TARGET.PARTIAL_SHIP = SOURCE.PARTIAL_SHIP,
TARGET.SHIPMENT_CARTONS = SOURCE.SHIPMENT_CARTONS,
TARGET.SCANNED_CARTONS = SOURCE.SCANNED_CARTONS,
TARGET.STAGED_CARTONS = SOURCE.STAGED_CARTONS,
TARGET.LOADED_CARTONS = SOURCE.LOADED_CARTONS,
TARGET.INVOICE_DATE = SOURCE.INVOICE_DATE,
TARGET.SHIPPING_LOAD_NUMBER = SOURCE.SHIPPING_LOAD_NUMBER,
TARGET.STARTING_WAVE = SOURCE.STARTING_WAVE,
TARGET.ENDING_WAVE = SOURCE.ENDING_WAVE,
TARGET.STAGED_DATE = SOURCE.STAGED_DATE,
TARGET.ORDER_HDR_STATUS = SOURCE.ORDER_HDR_STATUS,
TARGET.DELIVERY_NUMBER = SOURCE.DELIVERY_NUMBER,
TARGET.NOT_SELECTED_DATE = SOURCE.NOT_SELECTED_DATE,
TARGET.NOT_SELECTED_TIME = SOURCE.NOT_SELECTED_TIME,
TARGET.NOT_SELECTED_REASON = SOURCE.NOT_SELECTED_REASON,
TARGET.EST_DEL_DATE = SOURCE.EST_DEL_DATE,
TARGET.LAST_UPDATED = GETDATE(),
TARGET.[KEY] = SOURCE.[KEY]
WHEN NOT MATCHED BY TARGET
THEN INSERT
(
[SOURCE],
DC_ORDER_NUMBER,
DCMS_ORDER_TYPE,
CUSTOMER_PURCHASE_ORDER_ID,
BILL_TO,
CUSTOMER_MASTER_RECORD_TYPE,
SHIP_TO,
SALES_ORDER,
MERCHANDISE_STYLE,
MERCHANDISE_SIZE_1,
MATERIAL,
QUALITY,
ORDER_QTY,
ORDER_SELECTED_QTY,
ORDER_SHIPPED_QTY,
RELEASED_QTY,
PICK_QTY,
PACKED_QTY,
RELEASED_CARTONS,
FULL_CASE_PICK_COUNT,
TOTE_COUNT,
SPECIAL_PROCESS_CODE_1,
SPECIAL_PROCESS_CODE_2,
SPECIAL_PROCESS_CODE_3,
ORDER_DETAIL_STATUS,
ORDER_CARRIER,
CARRIER_SERVICE_ID,
CREATE_DATE,
CREATE_TIME,
ALLOCATION_DATE,
REQUESTED_SHIP_DATE,
CANCEL_DATE,
DISPATCH_DATE,
RELEASED_DATE,
RELEASED_TIME,
PRIORITY_ORDER,
PARTIAL_SHIP,
SHIPMENT_CARTONS,
SCANNED_CARTONS,
STAGED_CARTONS,
LOADED_CARTONS,
INVOICE_DATE,
SHIPPING_LOAD_NUMBER,
STARTING_WAVE,
ENDING_WAVE,
STAGED_DATE,
ORDER_HDR_STATUS,
DELIVERY_NUMBER,
NOT_SELECTED_DATE,
NOT_SELECTED_TIME,
NOT_SELECTED_REASON,
EST_DEL_DATE,
LAST_UPDATED,
[KEY]
)
VALUES
(
SOURCE.SOURCE,
SOURCE.DC_ORDER_NUMBER,
SOURCE.DCMS_ORDER_TYPE,
SOURCE.CUSTOMER_PURCHASE_ORDER_ID,
SOURCE.BILL_TO,
SOURCE.CUSTOMER_MASTER_RECORD_TYPE,
SOURCE.SHIP_TO,
SOURCE.SALES_ORDER,
SOURCE.MERCHANDISE_STYLE,
SOURCE.MERCHANDISE_SIZE_1,
SOURCE.MATERIAL,
SOURCE.QUALITY,
SOURCE.ORDER_QTY,
SOURCE.ORDER_SELECTED_QTY,
SOURCE.ORDER_SHIPPED_QTY,
SOURCE.RELEASED_QTY,
SOURCE.PICK_QTY,
SOURCE.PACKED_QTY,
SOURCE.RELEASED_CARTONS,
SOURCE.FULL_CASE_PICK_COUNT,
SOURCE.TOTE_COUNT,
SOURCE.SPECIAL_PROCESS_CODE_1,
SOURCE.SPECIAL_PROCESS_CODE_2,
SOURCE.SPECIAL_PROCESS_CODE_3,
SOURCE.ORDER_DETAIL_STATUS,
SOURCE.ORDER_CARRIER,
SOURCE.CARRIER_SERVICE_ID,
SOURCE.CREATE_DATE,
SOURCE.CREATE_TIME,
SOURCE.ALLOCATION_DATE,
SOURCE.REQUESTED_SHIP_DATE,
SOURCE.CANCEL_DATE,
SOURCE.DISPATCH_DATE,
SOURCE.RELEASED_DATE,
SOURCE.RELEASED_TIME,
SOURCE.PRIORITY_ORDER,
SOURCE.PARTIAL_SHIP,
SOURCE.SHIPMENT_CARTONS,
SOURCE.SCANNED_CARTONS,
SOURCE.STAGED_CARTONS,
SOURCE.LOADED_CARTONS,
SOURCE.INVOICE_DATE,
SOURCE.SHIPPING_LOAD_NUMBER,
SOURCE.STARTING_WAVE,
SOURCE.ENDING_WAVE,
SOURCE.STAGED_DATE,
SOURCE.ORDER_HDR_STATUS,
SOURCE.DELIVERY_NUMBER,
SOURCE.NOT_SELECTED_DATE,
SOURCE.NOT_SELECTED_TIME,
SOURCE.NOT_SELECTED_REASON,
SOURCE.EST_DEL_DATE,
GETDATE(),
SOURCE.[KEY]
);
END
【问题讨论】:
-
好的,我已经添加了执行计划的链接。谢谢!
-
71% 的成本是 FullOrderHeader 的表扫描 - 这表明,除非您查询需要所有这些,否则索引是有序的。
-
您应该尝试将远程表本地复制到 tempdb 中,而不是使用 cte。我怀疑你的主表也可以使用列存储
-
@DaleK 感谢您的想法。我减少了 UPDATE 语句中的列数,我意识到其中至少有一半没有更新,所以我把它们去掉了。将我的查询时间缩短了 5 分钟!!巨大差距。谢谢你的想法
标签: sql sql-server query-optimization