Data Refresh

Load Libraries

import eia_api as api
import eia_data 
import eia_impute as impute
import pandas as pd
import numpy as np
import requests
import json
import os
import datetime
import plotly.express as px
import plotly.graph_objects as go
from great_tables import GT

Settings

Load settings:

raw_json = open("../settings/settings.json")
meta_json = json.load(raw_json)
# Prototype version
# series = pd.DataFrame(meta_json["series_prototype"])
# Full version
series = pd.DataFrame(meta_json["series"])
api_path = meta_json["api_path"]
meta_path = meta_json["meta_path"]
data_path = meta_json["data_path"]
# Should be setting as false during development
save_data = True
save_meta = True

eia_api_key = os.getenv('EIA_API_KEY')
forecast_settings = meta_json["forecast"]
leaderboard_path = meta_json["backtesting"]["leaderboard_path"]
facets_template = {
    "parent" : None,
    "subba" : None
}

offset = meta_json["refresh"]["offset"]
metadata = api.eia_metadata(api_key = eia_api_key, api_path = api_path)

print(metadata.meta.keys())
print(metadata.meta["startPeriod"])
print(metadata.meta["endPeriod"])
dict_keys(['id', 'name', 'description', 'frequency', 'facets', 'data', 'startPeriod', 'endPeriod', 'defaultDateFormat', 'defaultFrequency'])
2019-01-01T00
2024-11-05T08
meta_obj = eia_data.get_metadata(api_key = eia_api_key, api_path = api_path, meta_path = meta_path, series = series, offset = 6)

GT(meta_obj.request_meta)
parent subba end_act request_start end updates_available
NYIS ZONA 2024-11-05 02:00:00 False
NYIS ZONB 2024-11-05 02:00:00 False
NYIS ZONC 2024-11-05 02:00:00 False
NYIS ZOND 2024-11-05 02:00:00 False
NYIS ZONE 2024-11-05 02:00:00 False
NYIS ZONF 2024-11-05 02:00:00 False
NYIS ZONG 2024-11-05 02:00:00 False
NYIS ZONH 2024-11-05 02:00:00 False
NYIS ZONI 2024-11-05 02:00:00 False
NYIS ZONJ 2024-11-05 02:00:00 False
NYIS ZONK 2024-11-05 02:00:00 False 
m = meta_obj.request_meta
index = meta_obj.last_index + 1
data = None
meta_new = None

for i in m.index:
    facets = facets_template
    facets["parent"] = m.at[i, "parent"]
    facets["subba"] = m.at[i, "subba"]
    start = m.at[i, "request_start"]
    end = m.at[i, "end"]

    # Case I - new data is available
    if m.at[i, "updates_available"]:
        print(m.at[i, "subba"], "- New updates are available, trying to refresh the data")
        temp = api.eia_backfill(api_key = eia_api_key, 
            api_path = api_path+ "data", 
            facets = facets, 
            start = start.to_pydatetime()- datetime.timedelta(hours = 24),
            end = end.to_pydatetime() + datetime.timedelta(hours = 24),
            offset = offset) 

        temp.data = temp.data[(temp.data["period"] <= end.to_pydatetime()) & (temp.data["period"] >= start.to_pydatetime())]

        end_actual = temp.data.period.max()
        ts_obj = pd.DataFrame(np.arange(start = start, stop = end_actual + datetime.timedelta(hours = 1), step = datetime.timedelta(hours = 1)).astype(datetime.datetime), columns=["index"])
        ts_obj  = ts_obj.merge(temp.data, left_on = "index", right_on = "period", how="left")
        ts_obj.drop("period", axis = 1, inplace= True)
        ts_obj = ts_obj.rename(columns= {"index": "period"})
    # Case II - new data is not available
    else:
        ts_obj = None
        print("No new data is available")

    meta_temp = eia_data.create_metadata(data = ts_obj, start = start, end = end, type = "refresh")
    print(meta_temp)
    meta_temp = pd.DataFrame([meta_temp])
    # Handling missing values
    if ts_obj is not None and meta_temp["na"].iloc[0] is not None and meta_temp["na"].iloc[0] >= 0:
        print(meta_temp)
        missing_index = ts_obj.value.isnull()
        ts_obj.loc[missing_index, "subba"] = meta_temp["subba"].iloc[0]
        ts_obj.loc[missing_index, "parent"] = meta_temp["parent"].iloc[0]
        ts = impute.impute_series(series= ts_obj, metadata =meta_temp)
        ts_obj = ts.data
        meta_temp = ts.metadata



    if ts_obj is None:
        meta_temp["parent"].iloc[0] =  m.at[i, "parent"]
        meta_temp["subba"].iloc[0] =  m.at[i, "subba"]
    # Append the data if the refresh was successful
    if meta_temp["success"].iloc[0] and meta_temp["update"].iloc[0]:
        print("Append the new data")
        d = eia_data.append_data(data_path = data_path, new_data = ts_obj, save = save_data)
        meta_temp["update"].iloc[0] = True
    elif not meta_temp["success"].iloc[0]:
        meta_temp["update"].iloc[0] = False
        meta_temp["comments"].iloc[0] = meta_temp["comments"].iloc[0] + "The data refresh failed, please check the log; "


    if data is None:
        data = ts_obj
    else:
        data = data._append(ts_obj)

    if meta_new is None:
        meta_new = meta_temp
    else:
        meta_new = meta_new._append(meta_temp)
meta_new.reset_index(drop=True, inplace=True)
No new data is available
{'index': None, 'parent': None, 'subba': None, 'time': datetime.datetime(2024, 11, 6, 8, 26, 37, 286815, tzinfo=datetime.timezone.utc), 'start': NaT, 'end': Timestamp('2024-11-05 02:00:00'), 'start_act': None, 'end_act': None, 'start_match': None, 'end_match': None, 'n_obs': None, 'na': None, 'imputed': 0, 'type': 'refresh', 'update': False, 'success': True, 'comments': 'No new data is available; '}
No new data is available
{'index': None, 'parent': None, 'subba': None, 'time': datetime.datetime(2024, 11, 6, 8, 26, 37, 289124, tzinfo=datetime.timezone.utc), 'start': NaT, 'end': Timestamp('2024-11-05 02:00:00'), 'start_act': None, 'end_act': None, 'start_match': None, 'end_match': None, 'n_obs': None, 'na': None, 'imputed': 0, 'type': 'refresh', 'update': False, 'success': True, 'comments': 'No new data is available; '}
No new data is available
{'index': None, 'parent': None, 'subba': None, 'time': datetime.datetime(2024, 11, 6, 8, 26, 37, 291813, tzinfo=datetime.timezone.utc), 'start': NaT, 'end': Timestamp('2024-11-05 02:00:00'), 'start_act': None, 'end_act': None, 'start_match': None, 'end_match': None, 'n_obs': None, 'na': None, 'imputed': 0, 'type': 'refresh', 'update': False, 'success': True, 'comments': 'No new data is available; '}
No new data is available
{'index': None, 'parent': None, 'subba': None, 'time': datetime.datetime(2024, 11, 6, 8, 26, 37, 293923, tzinfo=datetime.timezone.utc), 'start': NaT, 'end': Timestamp('2024-11-05 02:00:00'), 'start_act': None, 'end_act': None, 'start_match': None, 'end_match': None, 'n_obs': None, 'na': None, 'imputed': 0, 'type': 'refresh', 'update': False, 'success': True, 'comments': 'No new data is available; '}
No new data is available
{'index': None, 'parent': None, 'subba': None, 'time': datetime.datetime(2024, 11, 6, 8, 26, 37, 295706, tzinfo=datetime.timezone.utc), 'start': NaT, 'end': Timestamp('2024-11-05 02:00:00'), 'start_act': None, 'end_act': None, 'start_match': None, 'end_match': None, 'n_obs': None, 'na': None, 'imputed': 0, 'type': 'refresh', 'update': False, 'success': True, 'comments': 'No new data is available; '}
No new data is available
{'index': None, 'parent': None, 'subba': None, 'time': datetime.datetime(2024, 11, 6, 8, 26, 37, 298015, tzinfo=datetime.timezone.utc), 'start': NaT, 'end': Timestamp('2024-11-05 02:00:00'), 'start_act': None, 'end_act': None, 'start_match': None, 'end_match': None, 'n_obs': None, 'na': None, 'imputed': 0, 'type': 'refresh', 'update': False, 'success': True, 'comments': 'No new data is available; '}
No new data is available
{'index': None, 'parent': None, 'subba': None, 'time': datetime.datetime(2024, 11, 6, 8, 26, 37, 300227, tzinfo=datetime.timezone.utc), 'start': NaT, 'end': Timestamp('2024-11-05 02:00:00'), 'start_act': None, 'end_act': None, 'start_match': None, 'end_match': None, 'n_obs': None, 'na': None, 'imputed': 0, 'type': 'refresh', 'update': False, 'success': True, 'comments': 'No new data is available; '}
No new data is available
{'index': None, 'parent': None, 'subba': None, 'time': datetime.datetime(2024, 11, 6, 8, 26, 37, 302149, tzinfo=datetime.timezone.utc), 'start': NaT, 'end': Timestamp('2024-11-05 02:00:00'), 'start_act': None, 'end_act': None, 'start_match': None, 'end_match': None, 'n_obs': None, 'na': None, 'imputed': 0, 'type': 'refresh', 'update': False, 'success': True, 'comments': 'No new data is available; '}
No new data is available
{'index': None, 'parent': None, 'subba': None, 'time': datetime.datetime(2024, 11, 6, 8, 26, 37, 303755, tzinfo=datetime.timezone.utc), 'start': NaT, 'end': Timestamp('2024-11-05 02:00:00'), 'start_act': None, 'end_act': None, 'start_match': None, 'end_match': None, 'n_obs': None, 'na': None, 'imputed': 0, 'type': 'refresh', 'update': False, 'success': True, 'comments': 'No new data is available; '}
No new data is available
{'index': None, 'parent': None, 'subba': None, 'time': datetime.datetime(2024, 11, 6, 8, 26, 37, 305286, tzinfo=datetime.timezone.utc), 'start': NaT, 'end': Timestamp('2024-11-05 02:00:00'), 'start_act': None, 'end_act': None, 'start_match': None, 'end_match': None, 'n_obs': None, 'na': None, 'imputed': 0, 'type': 'refresh', 'update': False, 'success': True, 'comments': 'No new data is available; '}
No new data is available
{'index': None, 'parent': None, 'subba': None, 'time': datetime.datetime(2024, 11, 6, 8, 26, 37, 306829, tzinfo=datetime.timezone.utc), 'start': NaT, 'end': Timestamp('2024-11-05 02:00:00'), 'start_act': None, 'end_act': None, 'start_match': None, 'end_match': None, 'n_obs': None, 'na': None, 'imputed': 0, 'type': 'refresh', 'update': False, 'success': True, 'comments': 'No new data is available; '}
/tmp/ipykernel_7367/2864535603.py:51: FutureWarning:

ChainedAssignmentError: behaviour will change in pandas 3.0!
You are setting values through chained assignment. Currently this works in certain cases, but when using Copy-on-Write (which will become the default behaviour in pandas 3.0) this will never work to update the original DataFrame or Series, because the intermediate object on which we are setting values will behave as a copy.
A typical example is when you are setting values in a column of a DataFrame, like:

df["col"][row_indexer] = value

Use `df.loc[row_indexer, "col"] = values` instead, to perform the assignment in a single step and ensure this keeps updating the original `df`.

See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy


/tmp/ipykernel_7367/2864535603.py:51: SettingWithCopyWarning:


A value is trying to be set on a copy of a slice from a DataFrame

See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy

/tmp/ipykernel_7367/2864535603.py:52: FutureWarning:

ChainedAssignmentError: behaviour will change in pandas 3.0!
You are setting values through chained assignment. Currently this works in certain cases, but when using Copy-on-Write (which will become the default behaviour in pandas 3.0) this will never work to update the original DataFrame or Series, because the intermediate object on which we are setting values will behave as a copy.
A typical example is when you are setting values in a column of a DataFrame, like:

df["col"][row_indexer] = value

Use `df.loc[row_indexer, "col"] = values` instead, to perform the assignment in a single step and ensure this keeps updating the original `df`.

See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy


/tmp/ipykernel_7367/2864535603.py:52: SettingWithCopyWarning:


A value is trying to be set on a copy of a slice from a DataFrame

See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy

/tmp/ipykernel_7367/2864535603.py:51: FutureWarning:

ChainedAssignmentError: behaviour will change in pandas 3.0!
You are setting values through chained assignment. Currently this works in certain cases, but when using Copy-on-Write (which will become the default behaviour in pandas 3.0) this will never work to update the original DataFrame or Series, because the intermediate object on which we are setting values will behave as a copy.
A typical example is when you are setting values in a column of a DataFrame, like:

df["col"][row_indexer] = value

Use `df.loc[row_indexer, "col"] = values` instead, to perform the assignment in a single step and ensure this keeps updating the original `df`.

See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy


/tmp/ipykernel_7367/2864535603.py:51: SettingWithCopyWarning:


A value is trying to be set on a copy of a slice from a DataFrame

See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy

/tmp/ipykernel_7367/2864535603.py:52: FutureWarning:

ChainedAssignmentError: behaviour will change in pandas 3.0!
You are setting values through chained assignment. Currently this works in certain cases, but when using Copy-on-Write (which will become the default behaviour in pandas 3.0) this will never work to update the original DataFrame or Series, because the intermediate object on which we are setting values will behave as a copy.
A typical example is when you are setting values in a column of a DataFrame, like:

df["col"][row_indexer] = value

Use `df.loc[row_indexer, "col"] = values` instead, to perform the assignment in a single step and ensure this keeps updating the original `df`.

See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy


/tmp/ipykernel_7367/2864535603.py:52: SettingWithCopyWarning:


A value is trying to be set on a copy of a slice from a DataFrame

See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy

/tmp/ipykernel_7367/2864535603.py:51: FutureWarning:

ChainedAssignmentError: behaviour will change in pandas 3.0!
You are setting values through chained assignment. Currently this works in certain cases, but when using Copy-on-Write (which will become the default behaviour in pandas 3.0) this will never work to update the original DataFrame or Series, because the intermediate object on which we are setting values will behave as a copy.
A typical example is when you are setting values in a column of a DataFrame, like:

df["col"][row_indexer] = value

Use `df.loc[row_indexer, "col"] = values` instead, to perform the assignment in a single step and ensure this keeps updating the original `df`.

See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy


/tmp/ipykernel_7367/2864535603.py:51: SettingWithCopyWarning:


A value is trying to be set on a copy of a slice from a DataFrame

See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy

/tmp/ipykernel_7367/2864535603.py:52: FutureWarning:

ChainedAssignmentError: behaviour will change in pandas 3.0!
You are setting values through chained assignment. Currently this works in certain cases, but when using Copy-on-Write (which will become the default behaviour in pandas 3.0) this will never work to update the original DataFrame or Series, because the intermediate object on which we are setting values will behave as a copy.
A typical example is when you are setting values in a column of a DataFrame, like:

df["col"][row_indexer] = value

Use `df.loc[row_indexer, "col"] = values` instead, to perform the assignment in a single step and ensure this keeps updating the original `df`.

See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy


/tmp/ipykernel_7367/2864535603.py:52: SettingWithCopyWarning:


A value is trying to be set on a copy of a slice from a DataFrame

See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy

/tmp/ipykernel_7367/2864535603.py:51: FutureWarning:

ChainedAssignmentError: behaviour will change in pandas 3.0!
You are setting values through chained assignment. Currently this works in certain cases, but when using Copy-on-Write (which will become the default behaviour in pandas 3.0) this will never work to update the original DataFrame or Series, because the intermediate object on which we are setting values will behave as a copy.
A typical example is when you are setting values in a column of a DataFrame, like:

df["col"][row_indexer] = value

Use `df.loc[row_indexer, "col"] = values` instead, to perform the assignment in a single step and ensure this keeps updating the original `df`.

See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy


/tmp/ipykernel_7367/2864535603.py:51: SettingWithCopyWarning:


A value is trying to be set on a copy of a slice from a DataFrame

See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy

/tmp/ipykernel_7367/2864535603.py:52: FutureWarning:

ChainedAssignmentError: behaviour will change in pandas 3.0!
You are setting values through chained assignment. Currently this works in certain cases, but when using Copy-on-Write (which will become the default behaviour in pandas 3.0) this will never work to update the original DataFrame or Series, because the intermediate object on which we are setting values will behave as a copy.
A typical example is when you are setting values in a column of a DataFrame, like:

df["col"][row_indexer] = value

Use `df.loc[row_indexer, "col"] = values` instead, to perform the assignment in a single step and ensure this keeps updating the original `df`.

See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy


/tmp/ipykernel_7367/2864535603.py:52: SettingWithCopyWarning:


A value is trying to be set on a copy of a slice from a DataFrame

See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy

/tmp/ipykernel_7367/2864535603.py:51: FutureWarning:

ChainedAssignmentError: behaviour will change in pandas 3.0!
You are setting values through chained assignment. Currently this works in certain cases, but when using Copy-on-Write (which will become the default behaviour in pandas 3.0) this will never work to update the original DataFrame or Series, because the intermediate object on which we are setting values will behave as a copy.
A typical example is when you are setting values in a column of a DataFrame, like:

df["col"][row_indexer] = value

Use `df.loc[row_indexer, "col"] = values` instead, to perform the assignment in a single step and ensure this keeps updating the original `df`.

See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy


/tmp/ipykernel_7367/2864535603.py:51: SettingWithCopyWarning:


A value is trying to be set on a copy of a slice from a DataFrame

See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy

/tmp/ipykernel_7367/2864535603.py:52: FutureWarning:

ChainedAssignmentError: behaviour will change in pandas 3.0!
You are setting values through chained assignment. Currently this works in certain cases, but when using Copy-on-Write (which will become the default behaviour in pandas 3.0) this will never work to update the original DataFrame or Series, because the intermediate object on which we are setting values will behave as a copy.
A typical example is when you are setting values in a column of a DataFrame, like:

df["col"][row_indexer] = value

Use `df.loc[row_indexer, "col"] = values` instead, to perform the assignment in a single step and ensure this keeps updating the original `df`.

See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy


/tmp/ipykernel_7367/2864535603.py:52: SettingWithCopyWarning:


A value is trying to be set on a copy of a slice from a DataFrame

See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy

/tmp/ipykernel_7367/2864535603.py:51: FutureWarning:

ChainedAssignmentError: behaviour will change in pandas 3.0!
You are setting values through chained assignment. Currently this works in certain cases, but when using Copy-on-Write (which will become the default behaviour in pandas 3.0) this will never work to update the original DataFrame or Series, because the intermediate object on which we are setting values will behave as a copy.
A typical example is when you are setting values in a column of a DataFrame, like:

df["col"][row_indexer] = value

Use `df.loc[row_indexer, "col"] = values` instead, to perform the assignment in a single step and ensure this keeps updating the original `df`.

See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy


/tmp/ipykernel_7367/2864535603.py:51: SettingWithCopyWarning:


A value is trying to be set on a copy of a slice from a DataFrame

See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy

/tmp/ipykernel_7367/2864535603.py:52: FutureWarning:

ChainedAssignmentError: behaviour will change in pandas 3.0!
You are setting values through chained assignment. Currently this works in certain cases, but when using Copy-on-Write (which will become the default behaviour in pandas 3.0) this will never work to update the original DataFrame or Series, because the intermediate object on which we are setting values will behave as a copy.
A typical example is when you are setting values in a column of a DataFrame, like:

df["col"][row_indexer] = value

Use `df.loc[row_indexer, "col"] = values` instead, to perform the assignment in a single step and ensure this keeps updating the original `df`.

See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy


/tmp/ipykernel_7367/2864535603.py:52: SettingWithCopyWarning:


A value is trying to be set on a copy of a slice from a DataFrame

See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy

/tmp/ipykernel_7367/2864535603.py:51: FutureWarning:

ChainedAssignmentError: behaviour will change in pandas 3.0!
You are setting values through chained assignment. Currently this works in certain cases, but when using Copy-on-Write (which will become the default behaviour in pandas 3.0) this will never work to update the original DataFrame or Series, because the intermediate object on which we are setting values will behave as a copy.
A typical example is when you are setting values in a column of a DataFrame, like:

df["col"][row_indexer] = value

Use `df.loc[row_indexer, "col"] = values` instead, to perform the assignment in a single step and ensure this keeps updating the original `df`.

See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy


/tmp/ipykernel_7367/2864535603.py:51: SettingWithCopyWarning:


A value is trying to be set on a copy of a slice from a DataFrame

See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy

/tmp/ipykernel_7367/2864535603.py:52: FutureWarning:

ChainedAssignmentError: behaviour will change in pandas 3.0!
You are setting values through chained assignment. Currently this works in certain cases, but when using Copy-on-Write (which will become the default behaviour in pandas 3.0) this will never work to update the original DataFrame or Series, because the intermediate object on which we are setting values will behave as a copy.
A typical example is when you are setting values in a column of a DataFrame, like:

df["col"][row_indexer] = value

Use `df.loc[row_indexer, "col"] = values` instead, to perform the assignment in a single step and ensure this keeps updating the original `df`.

See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy


/tmp/ipykernel_7367/2864535603.py:52: SettingWithCopyWarning:


A value is trying to be set on a copy of a slice from a DataFrame

See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy

/tmp/ipykernel_7367/2864535603.py:51: FutureWarning:

ChainedAssignmentError: behaviour will change in pandas 3.0!
You are setting values through chained assignment. Currently this works in certain cases, but when using Copy-on-Write (which will become the default behaviour in pandas 3.0) this will never work to update the original DataFrame or Series, because the intermediate object on which we are setting values will behave as a copy.
A typical example is when you are setting values in a column of a DataFrame, like:

df["col"][row_indexer] = value

Use `df.loc[row_indexer, "col"] = values` instead, to perform the assignment in a single step and ensure this keeps updating the original `df`.

See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy


/tmp/ipykernel_7367/2864535603.py:51: SettingWithCopyWarning:


A value is trying to be set on a copy of a slice from a DataFrame

See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy

/tmp/ipykernel_7367/2864535603.py:52: FutureWarning:

ChainedAssignmentError: behaviour will change in pandas 3.0!
You are setting values through chained assignment. Currently this works in certain cases, but when using Copy-on-Write (which will become the default behaviour in pandas 3.0) this will never work to update the original DataFrame or Series, because the intermediate object on which we are setting values will behave as a copy.
A typical example is when you are setting values in a column of a DataFrame, like:

df["col"][row_indexer] = value

Use `df.loc[row_indexer, "col"] = values` instead, to perform the assignment in a single step and ensure this keeps updating the original `df`.

See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy


/tmp/ipykernel_7367/2864535603.py:52: SettingWithCopyWarning:


A value is trying to be set on a copy of a slice from a DataFrame

See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy

/tmp/ipykernel_7367/2864535603.py:51: FutureWarning:

ChainedAssignmentError: behaviour will change in pandas 3.0!
You are setting values through chained assignment. Currently this works in certain cases, but when using Copy-on-Write (which will become the default behaviour in pandas 3.0) this will never work to update the original DataFrame or Series, because the intermediate object on which we are setting values will behave as a copy.
A typical example is when you are setting values in a column of a DataFrame, like:

df["col"][row_indexer] = value

Use `df.loc[row_indexer, "col"] = values` instead, to perform the assignment in a single step and ensure this keeps updating the original `df`.

See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy


/tmp/ipykernel_7367/2864535603.py:51: SettingWithCopyWarning:


A value is trying to be set on a copy of a slice from a DataFrame

See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy

/tmp/ipykernel_7367/2864535603.py:52: FutureWarning:

ChainedAssignmentError: behaviour will change in pandas 3.0!
You are setting values through chained assignment. Currently this works in certain cases, but when using Copy-on-Write (which will become the default behaviour in pandas 3.0) this will never work to update the original DataFrame or Series, because the intermediate object on which we are setting values will behave as a copy.
A typical example is when you are setting values in a column of a DataFrame, like:

df["col"][row_indexer] = value

Use `df.loc[row_indexer, "col"] = values` instead, to perform the assignment in a single step and ensure this keeps updating the original `df`.

See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy


/tmp/ipykernel_7367/2864535603.py:52: SettingWithCopyWarning:


A value is trying to be set on a copy of a slice from a DataFrame

See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy

/tmp/ipykernel_7367/2864535603.py:51: FutureWarning:

ChainedAssignmentError: behaviour will change in pandas 3.0!
You are setting values through chained assignment. Currently this works in certain cases, but when using Copy-on-Write (which will become the default behaviour in pandas 3.0) this will never work to update the original DataFrame or Series, because the intermediate object on which we are setting values will behave as a copy.
A typical example is when you are setting values in a column of a DataFrame, like:

df["col"][row_indexer] = value

Use `df.loc[row_indexer, "col"] = values` instead, to perform the assignment in a single step and ensure this keeps updating the original `df`.

See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy


/tmp/ipykernel_7367/2864535603.py:51: SettingWithCopyWarning:


A value is trying to be set on a copy of a slice from a DataFrame

See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy

/tmp/ipykernel_7367/2864535603.py:52: FutureWarning:

ChainedAssignmentError: behaviour will change in pandas 3.0!
You are setting values through chained assignment. Currently this works in certain cases, but when using Copy-on-Write (which will become the default behaviour in pandas 3.0) this will never work to update the original DataFrame or Series, because the intermediate object on which we are setting values will behave as a copy.
A typical example is when you are setting values in a column of a DataFrame, like:

df["col"][row_indexer] = value

Use `df.loc[row_indexer, "col"] = values` instead, to perform the assignment in a single step and ensure this keeps updating the original `df`.

See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy


/tmp/ipykernel_7367/2864535603.py:52: SettingWithCopyWarning:


A value is trying to be set on a copy of a slice from a DataFrame

See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy

GT(meta_new,rowname_col = "index")
parent subba time start end start_act end_act start_match end_match n_obs na imputed type update success comments
NYIS ZONA 2024-11-06 08:26:37.286815+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
NYIS ZONB 2024-11-06 08:26:37.289124+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
NYIS ZONC 2024-11-06 08:26:37.291813+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
NYIS ZOND 2024-11-06 08:26:37.293923+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
NYIS ZONE 2024-11-06 08:26:37.295706+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
NYIS ZONF 2024-11-06 08:26:37.298015+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
NYIS ZONG 2024-11-06 08:26:37.300227+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
NYIS ZONH 2024-11-06 08:26:37.302149+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
NYIS ZONI 2024-11-06 08:26:37.303755+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
NYIS ZONJ 2024-11-06 08:26:37.305286+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
NYIS ZONK 2024-11-06 08:26:37.306829+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available; 
meta_updated = eia_data.append_metadata(meta_path = meta_path, meta = meta_new, save = save_meta, init = False)

GT(meta_updated)
/workspaces/pydata-ny-ga-workshop/pipeline/eia_data.py:66: FutureWarning:

The behavior of DataFrame concatenation with empty or all-NA entries is deprecated. In a future version, this will no longer exclude empty or all-NA columns when determining the result dtypes. To retain the old behavior, exclude the relevant entries before the concat operation.
index parent subba time start end start_act end_act start_match end_match n_obs na imputed type update success comments
1 NYIS ZONA 2024-11-05 05:30:28.064184+00:00 2022-01-01 00:00:00 2024-10-01 01:00:00 2022-01-01 00:00:00 2024-10-01 01:00:00 True True 24098.0 1.0 1 backfill True True Missing values were found; Missing values were imputed
1 NYIS ZONB 2024-11-05 05:30:39.101353+00:00 2022-01-01 00:00:00 2024-10-01 01:00:00 2022-01-01 00:00:00 2024-10-01 01:00:00 True True 24098.0 1.0 1 backfill True True Missing values were found; Missing values were imputed
1 NYIS ZONC 2024-11-05 05:30:54.164038+00:00 2022-01-01 00:00:00 2024-10-01 01:00:00 2022-01-01 00:00:00 2024-10-01 01:00:00 True True 24098.0 1.0 1 backfill True True Missing values were found; Missing values were imputed
1 NYIS ZOND 2024-11-05 05:31:09.183913+00:00 2022-01-01 00:00:00 2024-10-01 01:00:00 2022-01-01 00:00:00 2024-10-01 01:00:00 True True 24098.0 1.0 1 backfill True True Missing values were found; Missing values were imputed
1 NYIS ZONE 2024-11-05 05:31:24.350373+00:00 2022-01-01 00:00:00 2024-10-01 01:00:00 2022-01-01 00:00:00 2024-10-01 01:00:00 True True 24098.0 1.0 1 backfill True True Missing values were found; Missing values were imputed
1 NYIS ZONF 2024-11-05 05:31:38.829825+00:00 2022-01-01 00:00:00 2024-10-01 01:00:00 2022-01-01 00:00:00 2024-10-01 01:00:00 True True 24098.0 1.0 1 backfill True True Missing values were found; Missing values were imputed
1 NYIS ZONG 2024-11-05 05:31:54.169900+00:00 2022-01-01 00:00:00 2024-10-01 01:00:00 2022-01-01 00:00:00 2024-10-01 01:00:00 True True 24098.0 1.0 1 backfill True True Missing values were found; Missing values were imputed
1 NYIS ZONH 2024-11-05 05:32:08.296707+00:00 2022-01-01 00:00:00 2024-10-01 01:00:00 2022-01-01 00:00:00 2024-10-01 01:00:00 True True 24098.0 1.0 1 backfill True True Missing values were found; Missing values were imputed
1 NYIS ZONI 2024-11-05 05:32:23.408402+00:00 2022-01-01 00:00:00 2024-10-01 01:00:00 2022-01-01 00:00:00 2024-10-01 01:00:00 True True 24098.0 1.0 1 backfill True True Missing values were found; Missing values were imputed
1 NYIS ZONJ 2024-11-05 05:32:39.046172+00:00 2022-01-01 00:00:00 2024-10-01 01:00:00 2022-01-01 00:00:00 2024-10-01 01:00:00 True True 24098.0 1.0 1 backfill True True Missing values were found; Missing values were imputed
1 NYIS ZONK 2024-11-05 05:32:56.003404+00:00 2022-01-01 00:00:00 2024-10-01 01:00:00 2022-01-01 00:00:00 2024-10-01 01:00:00 True True 24098.0 1.0 1 backfill True True Missing values were found; Missing values were imputed
2 NYIS ZONA 2024-11-05 05:34:07.265687+00:00 2024-10-01 02:00:00 2024-11-04 02:00:00 2024-10-01 02:00:00 2024-11-04 02:00:00 True True 817.0 2.0 2 refresh True True Missing values were found; Missing values were imputed
2 NYIS ZONB 2024-11-05 05:34:08.968823+00:00 2024-10-01 02:00:00 2024-11-04 02:00:00 2024-10-01 02:00:00 2024-11-04 02:00:00 True True 817.0 2.0 2 refresh True True Missing values were found; Missing values were imputed
3 NYIS ZONA 2024-11-05 05:34:55.831717+00:00 2024-11-04 03:00:00 2024-11-04 02:00:00 0 refresh False True No new data is available;
3 NYIS ZONB 2024-11-05 05:34:55.832984+00:00 2024-11-04 03:00:00 2024-11-04 02:00:00 0 refresh False True No new data is available;
3 NYIS ZONC 2024-11-05 05:34:56.825905+00:00 2024-10-01 02:00:00 2024-11-04 02:00:00 2024-10-01 02:00:00 2024-11-04 02:00:00 True True 817.0 2.0 2 refresh True True Missing values were found; Missing values were imputed
3 NYIS ZOND 2024-11-05 05:34:58.683314+00:00 2024-10-01 02:00:00 2024-11-04 02:00:00 2024-10-01 02:00:00 2024-11-04 02:00:00 True True 817.0 2.0 2 refresh True True Missing values were found; Missing values were imputed
3 NYIS ZONE 2024-11-05 05:35:00.518441+00:00 2024-10-01 02:00:00 2024-11-04 02:00:00 2024-10-01 02:00:00 2024-11-04 02:00:00 True True 817.0 2.0 2 refresh True True Missing values were found; Missing values were imputed
3 NYIS ZONF 2024-11-05 05:35:02.446782+00:00 2024-10-01 02:00:00 2024-11-04 02:00:00 2024-10-01 02:00:00 2024-11-04 02:00:00 True True 817.0 2.0 2 refresh True True Missing values were found; Missing values were imputed
3 NYIS ZONG 2024-11-05 05:35:04.614086+00:00 2024-10-01 02:00:00 2024-11-04 02:00:00 2024-10-01 02:00:00 2024-11-04 02:00:00 True True 817.0 2.0 2 refresh True True Missing values were found; Missing values were imputed
3 NYIS ZONH 2024-11-05 05:35:06.207796+00:00 2024-10-01 02:00:00 2024-11-04 02:00:00 2024-10-01 02:00:00 2024-11-04 02:00:00 True True 817.0 2.0 2 refresh True True Missing values were found; Missing values were imputed
3 NYIS ZONI 2024-11-05 05:35:08.097463+00:00 2024-10-01 02:00:00 2024-11-04 02:00:00 2024-10-01 02:00:00 2024-11-04 02:00:00 True True 817.0 2.0 2 refresh True True Missing values were found; Missing values were imputed
3 NYIS ZONJ 2024-11-05 05:35:09.823799+00:00 2024-10-01 02:00:00 2024-11-04 02:00:00 2024-10-01 02:00:00 2024-11-04 02:00:00 True True 817.0 2.0 2 refresh True True Missing values were found; Missing values were imputed
3 NYIS ZONK 2024-11-05 05:35:11.438723+00:00 2024-10-01 02:00:00 2024-11-04 02:00:00 2024-10-01 02:00:00 2024-11-04 02:00:00 True True 817.0 2.0 2 refresh True True Missing values were found; Missing values were imputed
4 NYIS ZONA 2024-11-05 06:18:53.921533+00:00 2024-11-04 02:00:00 0 refresh False True No new data is available;
4 NYIS ZONB 2024-11-05 06:18:53.923160+00:00 2024-11-04 02:00:00 0 refresh False True No new data is available;
4 NYIS ZONC 2024-11-05 06:18:53.925576+00:00 2024-11-04 03:00:00 2024-11-04 02:00:00 0 refresh False True No new data is available;
4 NYIS ZOND 2024-11-05 06:18:53.927755+00:00 2024-11-04 03:00:00 2024-11-04 02:00:00 0 refresh False True No new data is available;
4 NYIS ZONE 2024-11-05 06:18:53.929798+00:00 2024-11-04 03:00:00 2024-11-04 02:00:00 0 refresh False True No new data is available;
4 NYIS ZONF 2024-11-05 06:18:53.931919+00:00 2024-11-04 03:00:00 2024-11-04 02:00:00 0 refresh False True No new data is available;
4 NYIS ZONG 2024-11-05 06:18:53.934277+00:00 2024-11-04 03:00:00 2024-11-04 02:00:00 0 refresh False True No new data is available;
4 NYIS ZONH 2024-11-05 06:18:53.936362+00:00 2024-11-04 03:00:00 2024-11-04 02:00:00 0 refresh False True No new data is available;
4 NYIS ZONI 2024-11-05 06:18:53.938384+00:00 2024-11-04 03:00:00 2024-11-04 02:00:00 0 refresh False True No new data is available;
4 NYIS ZONJ 2024-11-05 06:18:53.940429+00:00 2024-11-04 03:00:00 2024-11-04 02:00:00 0 refresh False True No new data is available;
4 NYIS ZONK 2024-11-05 06:18:53.942428+00:00 2024-11-04 03:00:00 2024-11-04 02:00:00 0 refresh False True No new data is available;
5 NYIS ZONA 2024-11-05 07:13:29.877656+00:00 2024-11-04 02:00:00 0 refresh False True No new data is available;
5 NYIS ZONB 2024-11-05 07:13:29.879325+00:00 2024-11-04 02:00:00 0 refresh False True No new data is available;
5 NYIS ZONC 2024-11-05 07:13:29.881743+00:00 2024-11-04 02:00:00 0 refresh False True No new data is available;
5 NYIS ZOND 2024-11-05 07:13:29.884192+00:00 2024-11-04 02:00:00 0 refresh False True No new data is available;
5 NYIS ZONE 2024-11-05 07:13:29.886783+00:00 2024-11-04 02:00:00 0 refresh False True No new data is available;
5 NYIS ZONF 2024-11-05 07:13:29.888828+00:00 2024-11-04 02:00:00 0 refresh False True No new data is available;
5 NYIS ZONG 2024-11-05 07:13:29.890905+00:00 2024-11-04 02:00:00 0 refresh False True No new data is available;
5 NYIS ZONH 2024-11-05 07:13:29.892945+00:00 2024-11-04 02:00:00 0 refresh False True No new data is available;
5 NYIS ZONI 2024-11-05 07:13:29.895023+00:00 2024-11-04 02:00:00 0 refresh False True No new data is available;
5 NYIS ZONJ 2024-11-05 07:13:29.897121+00:00 2024-11-04 02:00:00 0 refresh False True No new data is available;
5 NYIS ZONK 2024-11-05 07:13:29.899284+00:00 2024-11-04 02:00:00 0 refresh False True No new data is available;
6 NYIS ZONA 2024-11-05 08:18:01.940488+00:00 2024-11-04 02:00:00 0 refresh False True No new data is available;
6 NYIS ZONB 2024-11-05 08:18:01.942347+00:00 2024-11-04 02:00:00 0 refresh False True No new data is available;
6 NYIS ZONC 2024-11-05 08:18:01.944807+00:00 2024-11-04 02:00:00 0 refresh False True No new data is available;
6 NYIS ZOND 2024-11-05 08:18:01.947207+00:00 2024-11-04 02:00:00 0 refresh False True No new data is available;
6 NYIS ZONE 2024-11-05 08:18:01.949409+00:00 2024-11-04 02:00:00 0 refresh False True No new data is available;
6 NYIS ZONF 2024-11-05 08:18:01.951737+00:00 2024-11-04 02:00:00 0 refresh False True No new data is available;
6 NYIS ZONG 2024-11-05 08:18:01.953904+00:00 2024-11-04 02:00:00 0 refresh False True No new data is available;
6 NYIS ZONH 2024-11-05 08:18:01.956112+00:00 2024-11-04 02:00:00 0 refresh False True No new data is available;
6 NYIS ZONI 2024-11-05 08:18:01.958653+00:00 2024-11-04 02:00:00 0 refresh False True No new data is available;
6 NYIS ZONJ 2024-11-05 08:18:01.960799+00:00 2024-11-04 02:00:00 0 refresh False True No new data is available;
6 NYIS ZONK 2024-11-05 08:18:01.962907+00:00 2024-11-04 02:00:00 0 refresh False True No new data is available;
7 NYIS ZONA 2024-11-05 09:14:43.716806+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
7 NYIS ZONB 2024-11-05 09:14:43.718650+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
7 NYIS ZONC 2024-11-05 09:14:43.721175+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
7 NYIS ZOND 2024-11-05 09:14:43.723692+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
7 NYIS ZONE 2024-11-05 09:14:43.725798+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
7 NYIS ZONF 2024-11-05 09:14:43.727901+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
7 NYIS ZONG 2024-11-05 09:14:43.730062+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
7 NYIS ZONH 2024-11-05 09:14:43.732151+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
7 NYIS ZONI 2024-11-05 09:14:43.734230+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
7 NYIS ZONJ 2024-11-05 09:14:43.736288+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
7 NYIS ZONK 2024-11-05 09:14:43.738332+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
8 NYIS ZONA 2024-11-05 10:15:57.067161+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
8 NYIS ZONB 2024-11-05 10:15:57.068935+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
8 NYIS ZONC 2024-11-05 10:15:57.071304+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
8 NYIS ZOND 2024-11-05 10:15:57.073544+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
8 NYIS ZONE 2024-11-05 10:15:57.075710+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
8 NYIS ZONF 2024-11-05 10:15:57.077878+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
8 NYIS ZONG 2024-11-05 10:15:57.080089+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
8 NYIS ZONH 2024-11-05 10:15:57.082255+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
8 NYIS ZONI 2024-11-05 10:15:57.084406+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
8 NYIS ZONJ 2024-11-05 10:15:57.086646+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
8 NYIS ZONK 2024-11-05 10:15:57.089170+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
9 NYIS ZONA 2024-11-05 11:12:08.669659+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
9 NYIS ZONB 2024-11-05 11:12:08.671241+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
9 NYIS ZONC 2024-11-05 11:12:08.673733+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
9 NYIS ZOND 2024-11-05 11:12:08.676055+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
9 NYIS ZONE 2024-11-05 11:12:08.678521+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
9 NYIS ZONF 2024-11-05 11:12:08.680923+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
9 NYIS ZONG 2024-11-05 11:12:08.683318+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
9 NYIS ZONH 2024-11-05 11:12:08.685817+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
9 NYIS ZONI 2024-11-05 11:12:08.688200+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
9 NYIS ZONJ 2024-11-05 11:12:08.690246+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
9 NYIS ZONK 2024-11-05 11:12:08.692351+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
10 NYIS ZONA 2024-11-05 12:24:08.631223+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
10 NYIS ZONB 2024-11-05 12:24:08.633017+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
10 NYIS ZONC 2024-11-05 12:24:08.635545+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
10 NYIS ZOND 2024-11-05 12:24:08.638218+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
10 NYIS ZONE 2024-11-05 12:24:08.640476+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
10 NYIS ZONF 2024-11-05 12:24:08.642837+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
10 NYIS ZONG 2024-11-05 12:24:08.645469+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
10 NYIS ZONH 2024-11-05 12:24:08.647570+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
10 NYIS ZONI 2024-11-05 12:24:08.649617+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
10 NYIS ZONJ 2024-11-05 12:24:08.651712+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
10 NYIS ZONK 2024-11-05 12:24:08.653801+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
11 NYIS ZONA 2024-11-05 13:20:54.776474+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
11 NYIS ZONB 2024-11-05 13:20:54.778407+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
11 NYIS ZONC 2024-11-05 13:20:54.780916+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
11 NYIS ZOND 2024-11-05 13:20:54.783925+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
11 NYIS ZONE 2024-11-05 13:20:54.786570+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
11 NYIS ZONF 2024-11-05 13:20:54.788869+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
11 NYIS ZONG 2024-11-05 13:20:54.791105+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
11 NYIS ZONH 2024-11-05 13:20:54.793378+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
11 NYIS ZONI 2024-11-05 13:20:54.795598+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
11 NYIS ZONJ 2024-11-05 13:20:54.798202+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
11 NYIS ZONK 2024-11-05 13:20:54.800371+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
12 NYIS ZONA 2024-11-05 14:13:26.554799+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
12 NYIS ZONB 2024-11-05 14:13:26.556981+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
12 NYIS ZONC 2024-11-05 14:13:26.559205+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
12 NYIS ZOND 2024-11-05 14:13:26.561313+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
12 NYIS ZONE 2024-11-05 14:13:26.563461+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
12 NYIS ZONF 2024-11-05 14:13:26.565620+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
12 NYIS ZONG 2024-11-05 14:13:26.567760+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
12 NYIS ZONH 2024-11-05 14:13:26.569827+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
12 NYIS ZONI 2024-11-05 14:13:26.571927+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
12 NYIS ZONJ 2024-11-05 14:13:26.573969+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
12 NYIS ZONK 2024-11-05 14:13:26.576055+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
13 NYIS ZONA 2024-11-05 15:14:36.321526+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
13 NYIS ZONB 2024-11-05 15:14:36.323020+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
13 NYIS ZONC 2024-11-05 15:14:36.325403+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
13 NYIS ZOND 2024-11-05 15:14:36.327829+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
13 NYIS ZONE 2024-11-05 15:14:36.329900+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
13 NYIS ZONF 2024-11-05 15:14:36.331971+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
13 NYIS ZONG 2024-11-05 15:14:36.334037+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
13 NYIS ZONH 2024-11-05 15:14:36.336106+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
13 NYIS ZONI 2024-11-05 15:14:36.338178+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
13 NYIS ZONJ 2024-11-05 15:14:36.340248+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
13 NYIS ZONK 2024-11-05 15:14:36.342316+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
14 NYIS ZONA 2024-11-05 16:17:41.167646+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
14 NYIS ZONB 2024-11-05 16:17:41.169418+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
14 NYIS ZONC 2024-11-05 16:17:41.172029+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
14 NYIS ZOND 2024-11-05 16:17:41.174510+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
14 NYIS ZONE 2024-11-05 16:17:41.177095+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
14 NYIS ZONF 2024-11-05 16:17:41.179533+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
14 NYIS ZONG 2024-11-05 16:17:41.181975+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
14 NYIS ZONH 2024-11-05 16:17:41.184685+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
14 NYIS ZONI 2024-11-05 16:17:41.186776+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
14 NYIS ZONJ 2024-11-05 16:17:41.188861+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
14 NYIS ZONK 2024-11-05 16:17:41.190930+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
15 NYIS ZONA 2024-11-05 17:14:49.406620+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
15 NYIS ZONB 2024-11-05 17:14:49.408338+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
15 NYIS ZONC 2024-11-05 17:14:49.411120+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
15 NYIS ZOND 2024-11-05 17:14:49.413735+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
15 NYIS ZONE 2024-11-05 17:14:49.415809+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
15 NYIS ZONF 2024-11-05 17:14:49.417846+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
15 NYIS ZONG 2024-11-05 17:14:49.419889+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
15 NYIS ZONH 2024-11-05 17:14:49.421904+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
15 NYIS ZONI 2024-11-05 17:14:49.423959+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
15 NYIS ZONJ 2024-11-05 17:14:49.425997+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
15 NYIS ZONK 2024-11-05 17:14:49.428068+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
16 NYIS ZONA 2024-11-05 18:17:55.743993+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
16 NYIS ZONB 2024-11-05 18:17:55.745596+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
16 NYIS ZONC 2024-11-05 18:17:55.747886+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
16 NYIS ZOND 2024-11-05 18:17:55.750673+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
16 NYIS ZONE 2024-11-05 18:17:55.753029+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
16 NYIS ZONF 2024-11-05 18:17:55.755127+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
16 NYIS ZONG 2024-11-05 18:17:55.757269+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
16 NYIS ZONH 2024-11-05 18:17:55.759390+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
16 NYIS ZONI 2024-11-05 18:17:55.761507+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
16 NYIS ZONJ 2024-11-05 18:17:55.763625+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
16 NYIS ZONK 2024-11-05 18:17:55.765748+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
17 NYIS ZONA 2024-11-05 19:12:09.666775+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
17 NYIS ZONB 2024-11-05 19:12:09.668731+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
17 NYIS ZONC 2024-11-05 19:12:09.671448+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
17 NYIS ZOND 2024-11-05 19:12:09.673967+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
17 NYIS ZONE 2024-11-05 19:12:09.676600+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
17 NYIS ZONF 2024-11-05 19:12:09.678737+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
17 NYIS ZONG 2024-11-05 19:12:09.680868+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
17 NYIS ZONH 2024-11-05 19:12:09.683023+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
17 NYIS ZONI 2024-11-05 19:12:09.685149+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
17 NYIS ZONJ 2024-11-05 19:12:09.687302+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
17 NYIS ZONK 2024-11-05 19:12:09.689431+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
18 NYIS ZONA 2024-11-05 20:16:03.042153+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
18 NYIS ZONB 2024-11-05 20:16:03.043904+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
18 NYIS ZONC 2024-11-05 20:16:03.046573+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
18 NYIS ZOND 2024-11-05 20:16:03.049251+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
18 NYIS ZONE 2024-11-05 20:16:03.051354+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
18 NYIS ZONF 2024-11-05 20:16:03.053499+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
18 NYIS ZONG 2024-11-05 20:16:03.055604+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
18 NYIS ZONH 2024-11-05 20:16:03.057761+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
18 NYIS ZONI 2024-11-05 20:16:03.059895+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
18 NYIS ZONJ 2024-11-05 20:16:03.062049+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
18 NYIS ZONK 2024-11-05 20:16:03.064202+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
19 NYIS ZONA 2024-11-05 21:13:15.594788+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
19 NYIS ZONB 2024-11-05 21:13:15.596407+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
19 NYIS ZONC 2024-11-05 21:13:15.598740+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
19 NYIS ZOND 2024-11-05 21:13:15.601034+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
19 NYIS ZONE 2024-11-05 21:13:15.603507+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
19 NYIS ZONF 2024-11-05 21:13:15.605550+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
19 NYIS ZONG 2024-11-05 21:13:15.607639+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
19 NYIS ZONH 2024-11-05 21:13:15.609684+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
19 NYIS ZONI 2024-11-05 21:13:15.611742+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
19 NYIS ZONJ 2024-11-05 21:13:15.613767+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
19 NYIS ZONK 2024-11-05 21:13:15.615929+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
20 NYIS ZONA 2024-11-05 22:13:51.342465+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
20 NYIS ZONB 2024-11-05 22:13:51.343996+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
20 NYIS ZONC 2024-11-05 22:13:51.346271+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
20 NYIS ZOND 2024-11-05 22:13:51.348573+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
20 NYIS ZONE 2024-11-05 22:13:51.351012+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
20 NYIS ZONF 2024-11-05 22:13:51.353109+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
20 NYIS ZONG 2024-11-05 22:13:51.355173+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
20 NYIS ZONH 2024-11-05 22:13:51.357279+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
20 NYIS ZONI 2024-11-05 22:13:51.359355+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
20 NYIS ZONJ 2024-11-05 22:13:51.361467+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
20 NYIS ZONK 2024-11-05 22:13:51.363565+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
21 NYIS ZONA 2024-11-05 23:15:14.764800+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
21 NYIS ZONB 2024-11-05 23:15:14.766421+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
21 NYIS ZONC 2024-11-05 23:15:14.768905+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
21 NYIS ZOND 2024-11-05 23:15:14.771310+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
21 NYIS ZONE 2024-11-05 23:15:14.773733+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
21 NYIS ZONF 2024-11-05 23:15:14.776191+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
21 NYIS ZONG 2024-11-05 23:15:14.778605+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
21 NYIS ZONH 2024-11-05 23:15:14.781308+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
21 NYIS ZONI 2024-11-05 23:15:14.783410+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
21 NYIS ZONJ 2024-11-05 23:15:14.785570+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
21 NYIS ZONK 2024-11-05 23:15:14.787659+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
22 NYIS ZONA 2024-11-06 01:00:52.344497+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
22 NYIS ZONB 2024-11-06 01:00:52.346500+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
22 NYIS ZONC 2024-11-06 01:00:52.348936+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
22 NYIS ZOND 2024-11-06 01:00:52.351350+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
22 NYIS ZONE 2024-11-06 01:00:52.353920+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
22 NYIS ZONF 2024-11-06 01:00:52.356177+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
22 NYIS ZONG 2024-11-06 01:00:52.358298+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
22 NYIS ZONH 2024-11-06 01:00:52.360494+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
22 NYIS ZONI 2024-11-06 01:00:52.362612+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
22 NYIS ZONJ 2024-11-06 01:00:52.364845+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
22 NYIS ZONK 2024-11-06 01:00:52.367130+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
23 NYIS ZONA 2024-11-06 02:17:33.070254+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
23 NYIS ZONB 2024-11-06 02:17:33.071765+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
23 NYIS ZONC 2024-11-06 02:17:33.074022+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
23 NYIS ZOND 2024-11-06 02:17:33.076265+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
23 NYIS ZONE 2024-11-06 02:17:33.078553+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
23 NYIS ZONF 2024-11-06 02:17:33.080952+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
23 NYIS ZONG 2024-11-06 02:17:33.083007+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
23 NYIS ZONH 2024-11-06 02:17:33.085091+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
23 NYIS ZONI 2024-11-06 02:17:33.087158+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
23 NYIS ZONJ 2024-11-06 02:17:33.089254+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
23 NYIS ZONK 2024-11-06 02:17:33.091314+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
24 NYIS ZONA 2024-11-06 03:17:09.827251+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
24 NYIS ZONB 2024-11-06 03:17:09.828743+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
24 NYIS ZONC 2024-11-06 03:17:09.831047+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
24 NYIS ZOND 2024-11-06 03:17:09.833197+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
24 NYIS ZONE 2024-11-06 03:17:09.835420+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
24 NYIS ZONF 2024-11-06 03:17:09.837628+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
24 NYIS ZONG 2024-11-06 03:17:09.839823+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
24 NYIS ZONH 2024-11-06 03:17:09.842241+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
24 NYIS ZONI 2024-11-06 03:17:09.844300+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
24 NYIS ZONJ 2024-11-06 03:17:09.846398+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
24 NYIS ZONK 2024-11-06 03:17:09.848466+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
25 NYIS ZONA 2024-11-06 04:17:38.987374+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
25 NYIS ZONB 2024-11-06 04:17:38.988961+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
25 NYIS ZONC 2024-11-06 04:17:38.991674+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
25 NYIS ZOND 2024-11-06 04:17:38.994021+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
25 NYIS ZONE 2024-11-06 04:17:38.996520+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
25 NYIS ZONF 2024-11-06 04:17:38.998589+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
25 NYIS ZONG 2024-11-06 04:17:39.000618+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
25 NYIS ZONH 2024-11-06 04:17:39.002688+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
25 NYIS ZONI 2024-11-06 04:17:39.004703+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
25 NYIS ZONJ 2024-11-06 04:17:39.006788+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
25 NYIS ZONK 2024-11-06 04:17:39.008826+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
26 NYIS ZONA 2024-11-06 05:14:11.187091+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
26 NYIS ZONB 2024-11-06 05:14:11.188768+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
26 NYIS ZONC 2024-11-06 05:14:11.191355+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
26 NYIS ZOND 2024-11-06 05:14:11.193841+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
26 NYIS ZONE 2024-11-06 05:14:11.196369+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
26 NYIS ZONF 2024-11-06 05:14:11.198415+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
26 NYIS ZONG 2024-11-06 05:14:11.200494+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
26 NYIS ZONH 2024-11-06 05:14:11.202513+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
26 NYIS ZONI 2024-11-06 05:14:11.204591+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
26 NYIS ZONJ 2024-11-06 05:14:11.206679+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
26 NYIS ZONK 2024-11-06 05:14:11.208777+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
27 NYIS ZONA 2024-11-06 06:19:25.777661+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
27 NYIS ZONB 2024-11-06 06:19:25.779152+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
27 NYIS ZONC 2024-11-06 06:19:25.781370+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
27 NYIS ZOND 2024-11-06 06:19:25.783595+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
27 NYIS ZONE 2024-11-06 06:19:25.785733+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
27 NYIS ZONF 2024-11-06 06:19:25.787942+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
27 NYIS ZONG 2024-11-06 06:19:25.790083+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
27 NYIS ZONH 2024-11-06 06:19:25.792218+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
27 NYIS ZONI 2024-11-06 06:19:25.794424+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
27 NYIS ZONJ 2024-11-06 06:19:25.796801+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
27 NYIS ZONK 2024-11-06 06:19:25.798902+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
28 NYIS ZONA 2024-11-06 07:13:52.290840+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
28 NYIS ZONB 2024-11-06 07:13:52.292330+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
28 NYIS ZONC 2024-11-06 07:13:52.294662+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
28 NYIS ZOND 2024-11-06 07:13:52.296879+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
28 NYIS ZONE 2024-11-06 07:13:52.299330+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
28 NYIS ZONF 2024-11-06 07:13:52.301383+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
28 NYIS ZONG 2024-11-06 07:13:52.303468+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
28 NYIS ZONH 2024-11-06 07:13:52.305577+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
28 NYIS ZONI 2024-11-06 07:13:52.307643+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
28 NYIS ZONJ 2024-11-06 07:13:52.309772+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
28 NYIS ZONK 2024-11-06 07:13:52.311832+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
29 NYIS ZONA 2024-11-06 08:18:21.589771+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
29 NYIS ZONB 2024-11-06 08:18:21.591253+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
29 NYIS ZONC 2024-11-06 08:18:21.593766+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
29 NYIS ZOND 2024-11-06 08:18:21.596064+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
29 NYIS ZONE 2024-11-06 08:18:21.598270+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
29 NYIS ZONF 2024-11-06 08:18:21.600879+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
29 NYIS ZONG 2024-11-06 08:18:21.602944+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
29 NYIS ZONH 2024-11-06 08:18:21.605013+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
29 NYIS ZONI 2024-11-06 08:18:21.607039+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
29 NYIS ZONJ 2024-11-06 08:18:21.609134+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
29 NYIS ZONK 2024-11-06 08:18:21.611190+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
30 NYIS ZONA 2024-11-06 08:26:37.286815+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
30 NYIS ZONB 2024-11-06 08:26:37.289124+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
30 NYIS ZONC 2024-11-06 08:26:37.291813+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
30 NYIS ZOND 2024-11-06 08:26:37.293923+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
30 NYIS ZONE 2024-11-06 08:26:37.295706+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
30 NYIS ZONF 2024-11-06 08:26:37.298015+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
30 NYIS ZONG 2024-11-06 08:26:37.300227+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
30 NYIS ZONH 2024-11-06 08:26:37.302149+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
30 NYIS ZONI 2024-11-06 08:26:37.303755+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
30 NYIS ZONJ 2024-11-06 08:26:37.305286+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;
30 NYIS ZONK 2024-11-06 08:26:37.306829+00:00 2024-11-05 02:00:00 0 refresh False True No new data is available;

Plot the Series

We will use Plotly to visualize the series:

if data is not None:
    input = data.sort_values(by = ["subba", "period"])
    p = px.line(input, x="period", y="value", color="subba")
    p.show()
else: 
    input = None
    print("No new data is available")
No new data is available

Forecast Refresh

Reload the data:

df = pd.read_csv(data_path)
ts = df[["period", "subba", "y"]].copy()
ts["ds"] = pd.to_datetime(ts["period"])
ts = ts[["ds", "subba", "y"]]
ts = ts.rename(columns={"subba":"unique_id"})
/tmp/ipykernel_7367/4105294232.py:1: DtypeWarning:

Columns (4,5,6,8) have mixed types. Specify dtype option on import or set low_memory=False.

Check if new data is available and update the forecast:

forecast_log = pd.read_csv(forecast_settings["forecast_log_path"])
forecast_log = pd.read_csv(forecast_settings["forecast_log_path"])
forecast_log["end"] = pd.to_datetime(forecast_log["end"])
models_list = pd.read_csv(leaderboard_path)

forecast_new = None
forecast_log_new = None
for i in forecast_log["unique_id"].unique():
    end_fc = forecast_log[forecast_log["unique_id"] == i]["end"].max()
    d = ts[ts["unique_id"] == i]
    end_filter = d["ds"].max().floor(freq = "d") - datetime.timedelta(hours = 1)
    if end_filter > end_fc:
        print("New data points are available to series:", i, "refresh the forecast")
        input = d[d["ds"] <= end_filter]
        label = models_list[models_list["unique_id"] == i]["label"][0]
        model = models_list[models_list["unique_id"] == i]["model"][0]
        m = models_settings[label].copy()
        m["args"]["models"] = [model + "()"]
        if m["type"] == "mlforecast":
            args = m["args"]
            print(args)
            if "lags" not in args.keys():
                args["lags"] = None
            if "date_features" not in args.keys():
                args["date_features"] = None
            if "target_transforms" not in args.keys():
                args["target_transforms"] = None
            
            forecast_utils.models_reformat(args["models"])
            
            mlf = MLForecast(
                models =args["models"], 
                freq= args["freq"],
                date_features = args["date_features"],
                target_transforms= args["target_transforms"],
                lags = args["lags"])
            
            mlf.fit(df = input,  fitted=True, 
                    prediction_intervals = PredictionIntervals(n_windows = forecast_settings["prediction_intervals"]["n_windows"], h = forecast_settings["prediction_intervals"]["h"], 
                    method = forecast_settings["prediction_intervals"]["method"] ))
            fc_raw = None
            fc = None
            fc_raw = mlf.predict(forecast_settings["h"], 
            level  = [forecast_settings["prediction_intervals"]["level"]])
            fc = forecast_utils.fc_to_long(fc = fc_raw, models = args["models"], level = forecast_settings["prediction_intervals"]["level"])
            fc["label"] = label
            forecast_label = str(t.date().strftime("%Y-%m-%d"))
            fc["forecast_label"] = forecast_label

            if fc is not None and len(fc) > 0:
                fc_log = {
                    "unique_id": id,
                    "model": model,
                    "label": label,
                    "forecast_label": forecast_label,
                    "start": fc["ds"].min(),
                    "end": fc["ds"].max(),
                    "n_obs": len(fc),
                    "h": forecast_settings["h"],
                    "refresh_time": datetime.datetime.now().strftime("%d/%m/%Y %H:%M:%S")
                }
                if fc_log["n_obs"] == fc_log["h"]:
                    fc_log["success"] = True
                else:
                    fc_log["success"] = False
            else:
                fc_log = {
                    "unique_id": id,
                    "model": model,
                    "label": forecast_label,
                    "forecast_label": forecast_label
                    }
                fc_log["start"] = None
                fc_log["end"] = None
                fc_log["n_obs"] = None
                fc_log["h"] = forecast_settings["h"]
                fc_log["refresh_time"] = datetime.datetime.now().strftime("%d/%m/%Y %H:%M:%S")
                fc_log["success"] = False
            fc_log["mape"] = None
            fc_log["rmse"] = None
            fc_log["coverage"] = None
            fc_log["score"] = False

            if forecast_log is None:
                forecast_log = pd.DataFrame([fc_log])
            else:
                forecast_log = pd.concat([forecast_log, pd.DataFrame([fc_log])])    

            if forecast is None:
                forecast = fc
            else:
                forecast = pd.concat([forecast, fc])

if forecast_new is not None:
    forecast_new = forecast_new.sort_values(by = ["forecast_label", "unique_id"])
    forecast_new.reset_index(drop=True, inplace=True)

    forecast_log_new = forecast_log_new.sort_values(by = ["forecast_label", "unique_id"])
    forecast_log_new.reset_index(drop=True, inplace=True)

    forecast_new.to_csv(forecast_settings["forecast_path"], index = False)
    forecast_log_new.to_csv(forecast_settings["forecast_log_path"], index = False)
forecast_log = pd.read_csv(forecast_settings["forecast_log_path"])
update = False
if any(forecast_log["score"] == False):
    forecast_log["end"] = pd.to_datetime(forecast_log["end"])
    forecast = pd.read_csv(forecast_settings["forecast_path"])
    forecast["ds"] = pd.to_datetime(forecast["ds"])

    for index, row in forecast_log.iterrows():
        id = row["unique_id"]
        forecast_label = row["forecast_label"]

        fc = forecast[(forecast["forecast_label"] == forecast_label) & (forecast["unique_id"] == id)]

        fc = fc.merge(ts, left_on = ["unique_id", "ds"], right_on = ["unique_id", "ds"], how="left")
        forecast_log.at[index, "mape"] = forecast_utils.mape(y = fc["y"], yhat = fc["forecast"])
        forecast_log.at[index,"rmse"] = forecast_utils.rmse(y = fc["y"], yhat = fc["forecast"])
        forecast_log.at[index, "coverage"] =  forecast_utils.coverage(y = fc["y"], lower = fc["lower"], upper = fc["upper"])
        if len(fc) == row["h"]:
            forecast_log.at[index,"score"] = True
        update = True

if update:
    forecast_log.to_csv(forecast_settings["forecast_log_path"], index = False)

GT(forecast_log)
unique_id model label forecast_label start end n_obs h refresh_time success mape rmse coverage score
ZONA LinearRegression model4 2024-11-01 2024-11-02 2024-11-02 23:00:00 24 24 06/11/2024 07:55:09 True 0.043652243835248 95.44281545897655 0.7083333333333334 True
ZONB XGBRegressor model4 2024-11-01 2024-11-02 2024-11-02 23:00:00 24 24 06/11/2024 07:55:10 True 0.0489702879754926 55.11185472410671 0.6666666666666666 True
ZONC LinearRegression model4 2024-11-01 2024-11-02 2024-11-02 23:00:00 24 24 06/11/2024 07:55:10 True 0.0294291776131267 54.89107179763394 0.9583333333333334 True
ZOND LGBMRegressor model4 2024-11-01 2024-11-02 2024-11-02 23:00:00 24 24 06/11/2024 07:55:11 True 0.0229545288051701 18.555757547214046 1.0 True
ZONE LinearRegression model4 2024-11-01 2024-11-02 2024-11-02 23:00:00 24 24 06/11/2024 07:55:12 True 0.0221764717319209 21.898557451386345 0.9583333333333334 True
ZONF LinearRegression model4 2024-11-01 2024-11-02 2024-11-02 23:00:00 24 24 06/11/2024 07:55:12 True 0.0268308066646414 40.53750100987647 1.0 True
ZONG LinearRegression model4 2024-11-01 2024-11-02 2024-11-02 23:00:00 24 24 06/11/2024 07:55:13 True 0.1035057362451028 105.17885990069207 0.75 True
ZONH XGBRegressor model4 2024-11-01 2024-11-02 2024-11-02 23:00:00 24 24 06/11/2024 07:55:14 True 0.1064112030516249 25.45401422957301 0.4583333333333333 True
ZONI XGBRegressor model4 2024-11-01 2024-11-02 2024-11-02 23:00:00 24 24 06/11/2024 07:55:15 True 0.0505628646804763 33.89463797766538 0.8333333333333334 True
ZONJ LGBMRegressor model4 2024-11-01 2024-11-02 2024-11-02 23:00:00 24 24 06/11/2024 07:55:16 True 0.0656605589287828 314.9741894798421 0.0833333333333333 True
ZONK LinearRegression model4 2024-11-01 2024-11-02 2024-11-02 23:00:00 24 24 06/11/2024 07:55:16 True 0.1367364850464813 270.2176330551692 0.3333333333333333 True
ZONA LinearRegression model4 2024-11-02 2024-11-03 2024-11-03 23:00:00 24 24 06/11/2024 07:55:17 True 0.0406913003603799 71.60527947080918 0.7083333333333334 True
ZONB XGBRegressor model4 2024-11-02 2024-11-03 2024-11-03 23:00:00 24 24 06/11/2024 07:55:18 True 0.0632724334151587 60.72460597309195 0.75 True
ZONC LinearRegression model4 2024-11-02 2024-11-03 2024-11-03 23:00:00 24 24 06/11/2024 07:55:18 True 0.1093787429724178 180.95626539971647 0.7083333333333334 True
ZOND LGBMRegressor model4 2024-11-02 2024-11-03 2024-11-03 23:00:00 24 24 06/11/2024 07:55:19 True 0.0300886974736746 23.02897283243766 0.9166666666666666 True
ZONE LinearRegression model4 2024-11-02 2024-11-03 2024-11-03 23:00:00 24 24 06/11/2024 07:55:19 True 0.1260882106253828 97.36083704095964 0.75 True
ZONF LinearRegression model4 2024-11-02 2024-11-03 2024-11-03 23:00:00 24 24 06/11/2024 07:55:20 True 0.073546742306513 93.86685356486336 0.7916666666666666 True
ZONG LinearRegression model4 2024-11-02 2024-11-03 2024-11-03 23:00:00 24 24 06/11/2024 07:55:20 True 0.0712675709324085 65.10305390846375 0.7083333333333334 True
ZONH XGBRegressor model4 2024-11-02 2024-11-03 2024-11-03 23:00:00 24 24 06/11/2024 07:55:21 True 0.0727322594079201 20.817034743164285 0.5 True
ZONI XGBRegressor model4 2024-11-02 2024-11-03 2024-11-03 23:00:00 24 24 06/11/2024 07:55:23 True 0.0225743780372122 13.925164828678495 1.0 True
ZONJ LGBMRegressor model4 2024-11-02 2024-11-03 2024-11-03 23:00:00 24 24 06/11/2024 07:55:23 True 0.0152176194269242 78.30077282677799 1.0 True
ZONK LinearRegression model4 2024-11-02 2024-11-03 2024-11-03 23:00:00 24 24 06/11/2024 07:55:24 True 0.0500792251687481 95.35071556561104 0.7916666666666666 True