Samuel

2,923
reputation
1
4
19

I love data.

BigQuery: mapping of to datsets by a start_value giving in the map table:

With tbl1 as (SELECT a from unnest(generate_array(0,1000*1000*1)) a ), # a huge list of values
map as (SELECT sin(b) as b,b*100 as start_value from unnest(generate_array(0,10*1000*1)) b ) #map start_value to sin(b)

Select a,
 LAST_VALUE(b IGNORE NULLS) OVER ( ORDER BY a,b is null ROWS BETWEEN  UNBOUNDED preceding  and CURRENT ROW ) b_new,
FROM 
    (
    Select a, null as b from tbl1
    union all select start_value, b from map
    )
qualify b is  null

to create this for two tables:

CREATE temp FUNCTION keys(input STRING)
RETURNS Array<String>
LANGUAGE js AS """  return Object.keys(JSON.parse(input)); """;


With tbl1 as (SELECT a as a from unnest(generate_array(0,1000*1*1)) a ), # a huge list of values
map as (SELECT sin(b) as b,b*100 as start_value from unnest(generate_array(0,10*1*1)) b ) #map start_value to sin(b)


#replace map and tbl1 below and run
#in the result replace tbl1_a, map_b, start_value
SELECT replace(replace(replace(concat("""Select KEY_SET,
""" , key1 ,""" ,
""" , regexp_replace(concat(",",key2), r",([^,]+)", r"""LAST_VALUE(\1 IGNORE NULLS) OVER win  as \1_new, 
 """) , """
 FROM
 (
 SELECT *tbl1_a*  as  KEY_SET, 
  """ , key1 ," , " , null2 , """
 FROM tbl1 # edit
UNION ALL
 SELECT *start_value* as KEY_SET,
  """ , null1 ," , " , key2 , """
 FROM map #edit
 )
qualify *map_b* is null
 window win as ( ORDER BY KEY_SET asc , *map_b* is null ROWS BETWEEN  UNBOUNDED preceding  and CURRENT ROW )
  """),
  "*tbl1_a*","a"),  #replace
  "*map_b*","b"),
  "*start_value*","start_value"),
 From(select concat("null as ",array_to_string(keys(to_json_string(A)),", null as ")) as null1,
array_to_string(keys(to_json_string(A)),",") as key1,
from tbl1 as A # replace tbl1
limit 1)
cross join 
(select concat("null as ",array_to_string(keys(to_json_string(A)),", null as ")) as null2,
array_to_string(keys(to_json_string(A)),",") as key2,
from map as A #replace map
limit 1)