【问题标题】:CTE query performance improvement (postgres 9.6)CTE 查询性能改进(postgres 9.6)
【发布时间】:2017-09-24 01:20:46
【问题描述】:

我在下面有一个查询,它可以满足需要,但速度非常慢。它在给定的时间段内返回(例如,在 2017-05-01 00:00:00 和 2017-05-01 01:00:00 之间,但可以是 1 秒到几天之间的任何时间)第一个和最后一个记录给定的时间间隔(例如 15 秒,但可以是 1 秒到几天的任何时间)。

速度太慢了。例如,对于 2017-05-01 00:00:00 到 2017-05-01 00:00:01 期间的 1 秒间隔,它在我的 i7 7700HQ 上运行 6 秒。对于 2017-05-01 00:00:00 到 2017-05-01 00:00:05 期间 1 的间隔,我从未看到结果!该数据库现在大约有 6000 万行。在生产中,它将达到 10 亿个,每月增加约 5000 万个。

查询:

WITH ranges as (
    SELECT dd as start_range, 
           dd + '15 seconds'::interval as end_range, 
           ROW_NUMBER() over () as grp
    FROM generate_series
            ( '2017-05-01 00:00:00'::timestamp 
            , '2017-05-01 01:00:00'::timestamp
            , '15 seconds'::interval) dd
), create_grp as (
    SELECT r.grp, r.start_range, r.end_range, p.*
    FROM prices p
    JOIN ranges r
      ON p.dt >= r.start_range
      AND p.dt < r.end_range
    WHERE instrument='EURGBP'
), minmax as ( 
   SELECT row_number() over (partition by grp
                             order by dt asc) as rn1,
          row_number() over (partition by grp
                             order by dt desc) as rn2,              
          create_grp.*
   FROM create_grp 
)
SELECT *,
       CASE WHEN rn1 = 1 and rn2 = 1 THEN 'first and last'
            WHEN rn1 = 1 THEN 'first'
            WHEN rn2 = 1 THEN 'last'
       END as row_position
FROM minmax
WHERE 
    1 IN (rn1, rn2) 
ORDER BY dt
;

架构:

CREATE TABLE public.prices
(
    uid uuid NOT NULL DEFAULT uuid_generate_v4(),
    instrument character varying COLLATE pg_catalog."default" NOT NULL,
    bid double precision NOT NULL,
    ask double precision NOT NULL,
    dt timestamp without time zone NOT NULL DEFAULT now(),
    CONSTRAINT prices_pkey PRIMARY KEY (uid)
)
WITH (
    OIDS = FALSE
)
TABLESPACE pg_default;

索引:

CREATE INDEX idx_dt_instrument
    ON public.prices USING btree
    (dt, instrument COLLATE pg_catalog."default")
    TABLESPACE pg_default;

CREATE INDEX idx_dt_instrument_bid_ask
    ON public.prices USING btree
    (dt, instrument COLLATE pg_catalog."default", bid, ask)
    TABLESPACE pg_default;

CREATE INDEX idx_instrument
    ON public.prices USING btree
    (instrument COLLATE pg_catalog."default")
    TABLESPACE pg_default;

从 2017-05-01 00:00:00 开始 5 秒的示例数据集:

"uid","instrument","bid","ask","dt"
"4ecaa607-3733-4aba-9093-abc8f59e1638","EURGBP","0.84331","0.8434","2017-05-01 00:00:00.031"
"d1a41847-4945-4cf4-a45f-781db977ae07","GBPJPY","143.949005","143.970993","2017-05-01 00:00:00.031"
"34972c12-899b-404a-b0de-bae0fd3f6733","GBPJPY","143.947998","143.970993","2017-05-01 00:00:00.056"
"384b8246-3eac-4826-b6d2-d6caaa364f81","GBPUSD","1.29311","1.29323","2017-05-01 00:00:00.066"
"d879b04d-a4ed-452e-9208-7dfff672e860","GBPJPY","143.947006","143.970993","2017-05-01 00:00:00.067"
"a9e735ec-30d9-4c9a-a28e-5e5553273372","GBPJPY","143.945999","143.970993","2017-05-01 00:00:00.079"
"ee40ee5f-d8ac-41ce-9f39-ae50ef15d02d","GBPJPY","143.947006","143.964005","2017-05-01 00:00:00.091"
"7b605c3b-121f-46c2-a3e4-297d187f0a28","GBPJPY","143.947006","143.968994","2017-05-01 00:00:00.115"
"ccb307b0-7fa3-4354-8707-1426eded49e8","GBPJPY","143.942001","143.968994","2017-05-01 00:00:00.205"
"206c339d-bc36-469d-82d1-c7ae74002f44","EURGBP","0.84332","0.8434","2017-05-01 00:00:00.206"
"bc581318-91c7-4c80-85e0-e06f7236b277","GBPJPY","143.944","143.968994","2017-05-01 00:00:00.206"
"1fabf850-9045-4beb-81ae-bfa3adada62e","GBPUSD","1.29311","1.29324","2017-05-01 00:00:00.208"
"06d40f9a-a47e-466d-aebf-97a0154bdc74","GBPJPY","143.942001","143.968994","2017-05-01 00:00:00.209"
"b3b7fac9-340f-4e3b-8946-7bdecc383191","GBPUSD","1.29311","1.29327","2017-05-01 00:00:00.211"
"b5c28955-b40f-446f-9f43-9d5e9f145c1b","EURGBP","0.84331","0.8434","2017-05-01 00:00:00.212"
"192a40d6-8001-42ea-9430-96e800f2d4e8","GBPJPY","143.942993","143.968994","2017-05-01 00:00:00.212"
"e98dbba7-8231-4fa3-926b-291eb22b0f87","GBPJPY","143.944","143.968994","2017-05-01 00:00:00.215"
"7c6e952d-d01f-4dac-a1df-6b32168246fd","GBPUSD","1.29311","1.29326","2017-05-01 00:00:00.216"
"c86ba29f-3edb-4147-ba99-d9dfe594b0ff","GBPUSD","1.29312","1.29327","2017-05-01 00:00:00.243"
"35ca131e-b714-462d-827f-b5bc593aa3e6","GBPJPY","143.942993","143.968994","2017-05-01 00:00:00.262"
"91fc6fc0-7af9-4036-8e0e-29d4a3bb3e43","EURGBP","0.8433","0.8434","2017-05-01 00:00:00.283"
"e71946e0-1859-461a-b3eb-0539584ac4dc","GBPJPY","143.944","143.968994","2017-05-01 00:00:00.296"
"7321eea8-2610-408b-8dbf-4087f01e8c6e","GBPJPY","143.947998","143.968994","2017-05-01 00:00:00.377"
"f146716d-cadf-4e2f-9f17-6e7e8c5f2175","GBPUSD","1.29312","1.29327","2017-05-01 00:00:00.38"
"b3d81295-8cd5-44e7-879c-f15476ffac21","GBPUSD","1.29312","1.29326","2017-05-01 00:00:00.391"
"c037e27b-a8f4-4ec3-8472-b0f72a58fd33","EURGBP","0.8433","0.8434","2017-05-01 00:00:00.413"
"dba0f8f5-f218-49ea-8ebf-132f3ecf8910","GBPJPY","143.947998","143.968994","2017-05-01 00:00:00.443"
"a08449e3-44aa-4fed-b8e9-bf1a6bfc35c5","EURGBP","0.8433","0.84339","2017-05-01 00:00:00.585"
"a0b7ba20-653f-46db-93e9-d1edd8972dba","GBPUSD","1.29312","1.29326","2017-05-01 00:00:00.588"
"c2855ce8-8c5b-4de7-a92b-186d928e8f31","GBPJPY","143.947998","143.968994","2017-05-01 00:00:00.591"
"01b19a70-3ce7-44c7-9abd-9945321fdfd0","EURGBP","0.8433","0.84339","2017-05-01 00:00:00.621"
"4518aa9d-1f76-428e-ace4-7dcffeb6aa22","GBPJPY","143.947998","143.968994","2017-05-01 00:00:00.796"
"e4d4bac4-dd02-4da3-b231-20bfa6424412","EURGBP","0.8433","0.84339","2017-05-01 00:00:00.907"
"e48d3721-3157-4033-bd4f-09baae0f989c","GBPUSD","1.29312","1.29326","2017-05-01 00:00:00.909"
"64a33b2d-c756-4a0a-823e-075143ae7263","GBPJPY","143.947998","143.968994","2017-05-01 00:00:00.913"
"e477c47d-efd1-44dd-8058-cac17e08bc5d","GBPUSD","1.29314","1.29327","2017-05-01 00:00:00.914"
"cf6d5341-f7fd-47bc-89f6-a5448f78fb99","EURGBP","0.84329","0.84339","2017-05-01 00:00:00.943"
"4caa8bb9-094e-48ca-8a9a-7b2dd23a4fb4","GBPJPY","143.947006","143.968994","2017-05-01 00:00:00.967"
"274b7f51-3b07-430d-bfc5-a0b1ce62a750","GBPUSD","1.29312","1.29327","2017-05-01 00:00:00.975"
"2e3c2cd6-2525-46b3-86c5-b88e48bb0138","GBPJPY","143.947998","143.968994","2017-05-01 00:00:01.076"
"bcb12b90-2795-4789-bfa2-8e9a494eaeb1","GBPUSD","1.29312","1.29326","2017-05-01 00:00:01.076"
"d63d6037-fa81-47cc-bd62-4655850c0f80","GBPUSD","1.29312","1.29327","2017-05-01 00:00:01.077"
"6dbf8d8e-37c8-4537-80b5-c9219f4356b1","EURGBP","0.8433","0.84339","2017-05-01 00:00:01.079"
"7e3b6eaf-22e1-4f87-a7c1-226d3ee76146","EURGBP","0.84329","0.84339","2017-05-01 00:00:01.08"
"63e451c5-b8c6-4b57-ac9e-2171bc1dfbfa","GBPJPY","143.947998","143.968994","2017-05-01 00:00:01.121"
"866316e7-90a2-4c80-9a38-b3a062837415","GBPJPY","143.949005","143.968994","2017-05-01 00:00:01.143"
"fb11e963-cd36-4cfc-89b3-1bba3b8595b5","GBPUSD","1.29312","1.29327","2017-05-01 00:00:01.156"
"ad57e34f-5cbe-4b79-8579-b2c77c83b50b","EURGBP","0.84329","0.84339","2017-05-01 00:00:01.249"
"46b1840e-e424-41c2-8c71-691b201183ab","GBPJPY","143.949005","143.968994","2017-05-01 00:00:01.259"
"c5fa5e09-46df-4ea4-8640-9cf18e1f8fcc","GBPUSD","1.29312","1.29327","2017-05-01 00:00:01.265"
"926d092d-601e-43ec-b398-3300b0345cfd","GBPJPY","143.949997","143.968994","2017-05-01 00:00:01.267"
"4ddfbc84-20a1-4281-86c2-f0a4e77d0152","EURGBP","0.84329","0.84339","2017-05-01 00:00:01.305"
"e75af139-51a2-4a0a-acc8-e0adfbe5472a","GBPUSD","1.29313","1.29327","2017-05-01 00:00:01.346"
"408bca2d-2d57-471b-b6a7-819a3257741f","GBPJPY","143.951004","143.968994","2017-05-01 00:00:01.348"
"53c6f444-bd76-4a28-a370-7af98d5fa9ec","GBPUSD","1.29313","1.29326","2017-05-01 00:00:01.359"
"cd002ef4-925e-469b-8f2f-848cadd5943f","EURGBP","0.84329","0.84339","2017-05-01 00:00:01.443"
"13587e44-b694-4ce9-a626-e592d28c507d","EURGBP","0.8433","0.84339","2017-05-01 00:00:01.45"
"aabbacaf-3f09-4313-b992-e8b8d91df7ad","GBPJPY","143.951004","143.968994","2017-05-01 00:00:01.461"
"6b2111ef-c285-4482-b93c-238f27522ca3","GBPUSD","1.29313","1.29326","2017-05-01 00:00:01.477"
"19507f29-149c-4fdf-a1fa-aac312ab8479","EURGBP","0.8433","0.84338","2017-05-01 00:00:01.506"
"916cc759-a536-449d-b825-8203ebea9bf8","GBPJPY","143.949997","143.968994","2017-05-01 00:00:01.649"
"1fdd2b35-fd44-4dbb-81df-b98514af7004","GBPUSD","1.29313","1.29326","2017-05-01 00:00:01.649"
"3e9fc214-3cc6-4991-9eb3-2f5a5e557e96","GBPUSD","1.29312","1.29326","2017-05-01 00:00:01.65"
"5da9a29b-0d8a-42b1-98b6-f89dd2893c77","EURGBP","0.8433","0.84338","2017-05-01 00:00:01.651"
"b87f85e3-fba5-4556-8ca5-52625d978d53","EURGBP","0.84329","0.84338","2017-05-01 00:00:01.652"
"75b7624c-b90c-40d4-a7fc-09df7da9f659","GBPJPY","143.949997","143.968994","2017-05-01 00:00:01.715"
"b076534f-b893-4b55-9d9a-e3bac6e77ab2","GBPUSD","1.29312","1.29325","2017-05-01 00:00:01.732"
"6464da85-9eb5-4548-bd5e-3331a69f2121","EURGBP","0.84329","0.84338","2017-05-01 00:00:01.817"
"f9937464-e36a-4c57-a212-2f32943307d3","EURGBP","0.8433","0.84338","2017-05-01 00:00:01.83"
"1c7848e5-c101-4a50-87fe-1f5e980dbb95","GBPJPY","143.949005","143.968994","2017-05-01 00:00:01.83"
"30ad5bfc-f968-4b49-9b22-80c12e801f37","GBPUSD","1.29312","1.29325","2017-05-01 00:00:01.847"
"94eb4b51-4901-4e56-873a-05f93ef35e64","EURGBP","0.8433","0.84338","2017-05-01 00:00:02.007"
"97ef7e05-0a2f-4d70-80d1-1d3c748ea70f","GBPJPY","143.949005","143.968994","2017-05-01 00:00:02.007"
"af9274d9-02d6-43fc-9336-c3f512e4bd78","GBPUSD","1.29312","1.29325","2017-05-01 00:00:02.008"
"78c12ba8-992a-469e-bdda-bef004596aa1","GBPUSD","1.29311","1.29325","2017-05-01 00:00:02.062"
"4e1e31a9-bfdc-4f56-b8e1-0c7e83852fe0","GBPUSD","1.29311","1.29324","2017-05-01 00:00:02.071"
"e4f1c459-f75b-4005-8dd1-00c572ded4f9","GBPUSD","1.29311","1.29324","2017-05-01 00:00:02.203"
"2c8b38ce-50de-43e7-ba63-b6d1d7a7f76c","GBPJPY","143.947998","143.968994","2017-05-01 00:00:02.244"
"438f4439-a259-421f-8f45-a0dc88d80e1d","GBPJPY","143.949997","143.968994","2017-05-01 00:00:02.257"
"f12b7e18-84f0-4686-9588-0ecfab8f8bf8","GBPUSD","1.29311","1.29324","2017-05-01 00:00:02.301"
"f335ee32-cd5e-4cd2-aabe-06c14985778f","GBPJPY","143.949997","143.968994","2017-05-01 00:00:02.394"
"3c55786e-f9b8-424b-9c32-0b4b432a553f","EURGBP","0.8433","0.84338","2017-05-01 00:00:02.404"
"53c3e013-b016-41bb-a3ed-48158640831a","GBPUSD","1.29311","1.29324","2017-05-01 00:00:02.417"
"a944e2b6-3700-4ecd-994d-cb5ab97cf5ad","EURGBP","0.8433","0.84339","2017-05-01 00:00:02.508"
"ab32781d-1502-4047-b320-c3f4cda389fd","GBPJPY","143.949997","143.968994","2017-05-01 00:00:02.605"
"24759950-e095-4bb7-be3d-fda725423589","EURGBP","0.84329","0.84339","2017-05-01 00:00:02.625"
"cd7940d9-8a30-4a12-9dc9-a2c710cbb982","GBPUSD","1.29311","1.29325","2017-05-01 00:00:02.649"
"233e0a3d-eeae-403f-b512-2add055d6735","GBPJPY","143.949997","143.968994","2017-05-01 00:00:02.761"
"58c7c33c-75b2-45bd-a50c-615d9eaaec07","EURGBP","0.84329","0.84339","2017-05-01 00:00:02.763"
"675b2847-b0f9-44f1-ab75-d6ddafbddfd5","GBPUSD","1.29311","1.29325","2017-05-01 00:00:02.766"
"d2946714-e84c-4a36-9819-0cca4f2dd197","EURGBP","0.8433","0.84339","2017-05-01 00:00:02.81"
"5322418b-4c45-4771-b591-cc8fea167424","GBPJPY","143.949997","143.968994","2017-05-01 00:00:02.832"
"9a1c572e-28b9-4d16-b196-ccef3e48f602","EURGBP","0.84331","0.84339","2017-05-01 00:00:02.929"
"a4377169-5400-4dd6-81f0-8b455d178b65","EURGBP","0.8433","0.84339","2017-05-01 00:00:02.951"
"37cdddb0-db5e-4574-bf4d-45fb8d52c6be","GBPUSD","1.29311","1.29325","2017-05-01 00:00:02.951"
"1258c886-b797-43ed-9886-323daac83dde","GBPUSD","1.29311","1.29325","2017-05-01 00:00:02.999"
"453406c3-5902-4dab-b39c-2e0b85739767","EURGBP","0.8433","0.84339","2017-05-01 00:00:03.005"
"95699a6d-8412-4f2f-9dd4-86a1d3eed57d","GBPUSD","1.29312","1.29325","2017-05-01 00:00:03.129"
"2268fcc4-104e-434b-8237-bce765bda084","EURGBP","0.8433","0.84339","2017-05-01 00:00:03.131"
"ba771b59-b04c-4683-8ceb-8e9b8451bcc7","GBPUSD","1.29312","1.29325","2017-05-01 00:00:03.17"
"00541afa-79a6-4d09-8c6e-4d59e8d15bb2","GBPJPY","143.947006","143.968994","2017-05-01 00:00:03.216"
"b4dd6e15-0e25-4da1-b210-021b91e4245a","GBPUSD","1.29312","1.29325","2017-05-01 00:00:03.323"
"79b11513-97c5-4146-a4c8-7daf7d8ffe6f","GBPJPY","143.947006","143.966995","2017-05-01 00:00:03.324"
"c5283502-eb09-45d3-b92b-d5625de9e9fc","GBPJPY","143.947006","143.966003","2017-05-01 00:00:03.411"
"67715855-9171-4ae7-b205-5ba017d328f2","EURGBP","0.8433","0.84339","2017-05-01 00:00:03.501"
"00b7647e-7d53-407d-9704-78a8712ac580","GBPJPY","143.947006","143.966003","2017-05-01 00:00:03.525"
"b41b5483-5fb2-4c57-9892-0d02e5e6a823","GBPJPY","143.947998","143.966003","2017-05-01 00:00:03.549"
"881712b2-ff9a-4065-ad6a-9caee03283e3","GBPJPY","143.947998","143.964996","2017-05-01 00:00:03.56"
"7248bc13-5a9f-4fba-97a1-fe041e975c38","EURGBP","0.8433","0.84339","2017-05-01 00:00:03.581"
"0379b4bb-7691-48e8-a668-2388f6e5d510","GBPJPY","143.947998","143.964996","2017-05-01 00:00:03.672"
"2715fe9a-eb6f-445d-b7d0-850293bb5b2e","GBPJPY","143.947006","143.964996","2017-05-01 00:00:03.698"
"52ce7685-0964-46e4-ab8f-282e68bdb73d","GBPJPY","143.947006","143.964005","2017-05-01 00:00:03.711"
"6b31dee4-1aec-4d10-9b37-0c58bae7e449","GBPJPY","143.947006","143.964005","2017-05-01 00:00:03.848"
"7e61571a-ea8b-44cb-9481-ae31ba8b86c4","GBPUSD","1.29312","1.29325","2017-05-01 00:00:03.868"
"89e316cb-e01d-4f01-9dcd-a380e94ed4e8","GBPJPY","143.947006","143.964005","2017-05-01 00:00:03.943"
"966318e4-a6b5-4c1b-82e1-fbbf0029ce02","GBPUSD","1.29312","1.29325","2017-05-01 00:00:04.096"
"8fc7b075-7f5e-40f5-9879-55604f77a3c5","EURGBP","0.8433","0.84339","2017-05-01 00:00:04.227"
"23ab8b66-337e-4c4e-b73a-f31101716565","GBPJPY","143.947006","143.964005","2017-05-01 00:00:04.227"
"eb7469be-8a40-4498-af85-7b2c42630149","GBPJPY","143.947006","143.964005","2017-05-01 00:00:04.288"
"f945a821-9790-4760-beea-1b54bb1c2a85","EURGBP","0.8433","0.84339","2017-05-01 00:00:04.406"
"978c6d4c-22f9-4218-b447-940a3d3c436e","EURGBP","0.8433","0.84339","2017-05-01 00:00:04.505"
"2ace7d77-b228-491c-9cf0-610b8e48cdf7","GBPJPY","143.947006","143.966003","2017-05-01 00:00:04.786"
"0f89d5dd-52f2-44e9-aa54-cbb423cd7416","GBPJPY","143.947006","143.964996","2017-05-01 00:00:04.787"
"db7748a8-317b-4c62-a1f2-d6679430b343","GBPUSD","1.29312","1.29325","2017-05-01 00:00:04.895"
"ae29728e-773b-42a9-9f1e-199ba996731a","EURGBP","0.8433","0.84339","2017-05-01 00:00:04.986"
"19ffa8e4-63c5-4f47-90d5-771dd5a839ba","GBPJPY","143.947006","143.962997","2017-05-01 00:00:04.986"

减速是由“ WHERE instrument='EURGBP'”引起的。一旦我删除它,查询就会飞起来。但是,我必须通过“仪器”进行过滤。

解释输出:

Sort  (cost=2356660289.95..2356711257.16 rows=20386886 width=144)
  Sort Key: minmax.dt
  CTE ranges
    ->  WindowAgg  (cost=0.00..25.00 rows=1000 width=24)
          ->  Function Scan on generate_series dd  (cost=0.00..10.00 rows=1000 width=8)
  CTE create_grp
    ->  Nested Loop  (cost=344299.48..460976348.77 rows=2043798111 width=71)
          Join Filter: ((p.dt >= r.start_range) AND (p.dt < r.end_range))
          ->  Bitmap Heap Scan on prices p  (cost=344299.48..1121763.77 rows=18394183 width=47)
                Recheck Cond: ((instrument)::text = 'EURGBP'::text)
                ->  Bitmap Index Scan on idx_instrument  (cost=0.00..339700.94 rows=18394183 width=0)
                      Index Cond: ((instrument)::text = 'EURGBP'::text)
          ->  CTE Scan on ranges r  (cost=0.00..20.00 rows=1000 width=24)
  CTE minmax
    ->  WindowAgg  (cost=1796644092.85..1837520055.07 rows=2043798111 width=112)
          ->  Sort  (cost=1796644092.85..1801753588.13 rows=2043798111 width=104)
                Sort Key: create_grp.grp, create_grp.dt DESC
                ->  WindowAgg  (cost=880857947.67..921733909.89 rows=2043798111 width=104)
                      ->  Sort  (cost=880857947.67..885967442.95 rows=2043798111 width=96)
                            Sort Key: create_grp.grp, create_grp.dt
                            ->  CTE Scan on create_grp  (cost=0.00..40875962.22 rows=2043798111 width=96)
  ->  CTE Scan on minmax  (cost=0.00..51298821.64 rows=20386886 width=144)
        Filter: ((1 = rn1) OR (1 = rn2))

非常欢迎任何优化它的建议。

【问题讨论】:

  • EXPLAIN 是否表明正在使用WHERE 子句上的索引?
  • 您已经在对原始问题的反应中被告知:查询速度很慢因为的 CTE 和 generate_series(),两者都缺少要加入的索引,和/或可用的统计数据。 IOW:将它们具体化。 (WHERE instrument='EURGBP' 条件是最强提示,导致规划者改变主意)
  • 我是整个 SQL 的新手。我可以在这里以某种方式使用临时表吗? “物化”是什么意思?我如何实现它们?
  • 顺便说一句:您的索引没有意义(一个是另一个的子集)并且将属性(bid,ask)添加到索引无论如何都是相对无用的,其他两个组件是 allmost 一个候选键)

标签: postgresql


【解决方案1】:
CREATE TEMP TABLE ranges
        ( start_range timestamp NOT NULL
        , end_range timestamp NOT NULL
        , grp INTEGER NOT NULL UNIQUE
        , PRIMARY KEY (start_range)
        );
INSERT INTO ranges(start_range, end_range, grp)
SELECT dd as start_range,
           dd + '1 seconds'::interval as end_range,
           ROW_NUMBER() over () as grp
    FROM generate_series
            ( '2017-05-01 00:00:00'::timestamp
            , '2017-05-01 01:00:00'::timestamp
            --, '15 seconds'::interval) dd
            , '1 seconds'::interval) dd
        ;

VACUUM ANALYZE ranges;

-- EXPLAIN ANALYZE
SELECT *,
       CASE WHEN rn1 = 1 and rn2 = 1 THEN 'first and last'
            WHEN rn1 = 1 THEN 'first'
            WHEN rn2 = 1 THEN 'last'
       END as row_position
FROM (
    SELECT r.grp, p.uid, p.instrument, r.start_range , p.dt AS dt
           , p.bid, p.ask
        , row_number() over (partition by r.grp order by p.dt asc) as rn1
        , row_number() over (partition by r.grp order by p.dt desc) as rn2
    FROM prices p
    JOIN ranges r ON p.dt >= r.start_range AND p.dt < r.end_range
    WHERE p.instrument='EURGBP'
        ) zzzz
WHERE (rn1=1 OR rn2=1)
-- ORDER BY instrument,grp, dt
        ;

而且,您实际上并不需要行号,因为您只需要窗口的开头/结尾:


-- EXPLAIN ANALYZE
SELECT *,
       CASE WHEN (prev IS NULL AND next IS NULL) THEN 'first and last'
            WHEN prev IS NULL THEN 'first'
            WHEN next IS NULL THEN 'last'
       END as row_position
FROM (
    SELECT r.grp, p.uid, p.instrument, r.start_range , p.dt AS dt, p.bid, p.ask
        , lag(uid) over (www) as prev
        , lead(uid) over (www) as next
    FROM prices p
    JOIN ranges r ON p.dt >= r.start_range AND p.dt < r.end_range
    WHERE p.instrument='EURGBP'
        WINDOW www AS (partition by r.grp order by p.dt )
        -- ORDER BY p.instrument,r.grp, p.dt
        ) qqqq
WHERE (prev IS NULL OR next IS NULL)
-- ORDER BY instrument,grp, dt
        ;

【讨论】:

  • 感谢重构。但是,您的方法比原来的还要慢。我不确定为什么。相同周期(1 秒)和相同间隔(1 秒)的原始查询在 10 秒内运行。你的 12 岁。我猜 60mills 的数据库大小很重要。
  • 调优很重要。基数很重要。索引很重要。
【解决方案2】:

感谢大家的帮助和提示!显然,如果我将 WHERE 子句从 create_gpr 移动到 minimax,查询会变得非常快。我对此没有任何解释。但它有效。这里是:

WITH ranges as (
    SELECT dd as start_range, 
           dd + '1 seconds'::interval as end_range, 
           ROW_NUMBER() over () as grp
    FROM generate_series
            ( '2017-05-01 00:00:00'::timestamp 
            , '2017-05-01 00:01:00'::timestamp
            , '1 seconds'::interval) dd
), create_grp as (
    SELECT r.grp, r.start_range, r.end_range, p.*
    FROM prices p
    JOIN ranges r
      ON p.dt >= r.start_range
     AND p.dt < r.end_range
-- WHERE need to be moved out of here, which has no indexes as being temporary
--  WHERE instrument='EURGBP'
), minmax as ( 
   SELECT row_number() over (partition by grp
                             order by dt asc) as rn1,
          row_number() over (partition by grp
                             order by dt desc) as rn2,              
          create_grp.*
   FROM create_grp
-- Here WHERE goes! It does use index here for some reason. And the query flies.
   WHERE instrument='EURGBP'
)
SELECT *,
       CASE WHEN rn1 = 1 and rn2 = 1 THEN 'first and last'
            WHEN rn1 = 1 THEN 'first'
            WHEN rn2 = 1 THEN 'last'
       END as row_position
FROM minmax
WHERE 
    1 IN (rn1, rn2) 
ORDER BY dt
;

这个查询(1 分钟范围)在 84 毫秒内运行,旧的从未完成运行(10 分钟后我放弃了)。旧查询在 1 秒的范围内运行了 10 秒。

感谢大家的帮助!

【讨论】:

    猜你喜欢
    • 1970-01-01
    • 2018-03-29
    • 1970-01-01
    • 2012-11-06
    • 2019-02-13
    • 1970-01-01
    • 2023-03-30
    • 1970-01-01
    • 1970-01-01
    相关资源
    最近更新 更多