Create a postgres function to devide-and-conquer (iterate) your big query
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 | TRUNCATE TABLE YOUR_NEW_BIG_TABLE; DROP FUNCTION IF EXISTS pg_iterator(); CREATE OR REPLACE FUNCTION pg_iterator() RETURNS void AS $BODY$ DECLARE vOffsetRecord INT; vTotal INT; vLimit INT; vOffset INT; BEGIN -- How many? -- select count(*) INTO currentRecord from YOUR_BIG_TABLE; -- 346,879,848 vTotal := 346879848; -- vTotal := 5000; vLimit := 100000; vOffset :=0; RAISE NOTICE 'DEBUG vTotal : %', vTotal; RAISE NOTICE 'DEBUG vLimit : %', vLimit; RAISE NOTICE 'DEBUG vOffset: %', vOffset; LOOP IF (vOffset >= vTotal) THEN EXIT; END IF; EXECUTE format('INSERT INTO YOUR_NEW_BIG_TABLE (id, language, hash) SELECT id, language, hash FROM YOUR_BIG_TABLE ORDER BY id LIMIT %s OFFSET %s;', vLimit, vOffset); RAISE NOTICE 'DEBUG vOffset: %', vOffset; vOffset := vOffset + vLimit; END LOOP; END$BODY$ LANGUAGE plpgsql VOLATILE COST 100; ALTER FUNCTION pg_iterator() OWNER TO postgres; SELECT pg_iterator(); |