Postgres function to devide-and-conquer (iterate) your big query

Create a postgres function to devide-and-conquer (iterate) your big query

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
TRUNCATE TABLE YOUR_NEW_BIG_TABLE;

DROP FUNCTION IF EXISTS pg_iterator();

CREATE OR REPLACE FUNCTION pg_iterator()
  RETURNS void AS
$BODY$
DECLARE
    vOffsetRecord INT;
    vTotal INT;
    vLimit INT;
    vOffset INT;
BEGIN
   
    -- How many?
    -- select count(*) INTO currentRecord from YOUR_BIG_TABLE; -- 346,879,848
    vTotal := 346879848;
    --  vTotal := 5000;
    vLimit  := 100000;
    vOffset :=0;
   
    RAISE NOTICE 'DEBUG vTotal : %', vTotal;
    RAISE NOTICE 'DEBUG vLimit : %', vLimit;
    RAISE NOTICE 'DEBUG vOffset: %', vOffset;

    LOOP
        IF (vOffset >=  vTotal) THEN
            EXIT;
        END IF;
        EXECUTE format('INSERT INTO YOUR_NEW_BIG_TABLE (id, language, hash) SELECT id, language, hash FROM YOUR_BIG_TABLE ORDER BY id LIMIT %s OFFSET %s;', vLimit, vOffset);
        RAISE NOTICE 'DEBUG vOffset: %', vOffset;
        vOffset := vOffset + vLimit;
    END LOOP;
   
END$BODY$
  LANGUAGE plpgsql VOLATILE
  COST 100;
ALTER FUNCTION pg_iterator()
  OWNER TO postgres;

SELECT pg_iterator();
This entry was posted in Database, PostgreSQL. Bookmark the permalink.

Leave a Reply

Your email address will not be published. Required fields are marked *


− 6 = one