Bug improve fred get source and reader methods

Demonstration.cs

@@ -0,0 +1,97 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System.Linq;
+using QuantConnect.Data;
+using QuantConnect.Util;
+using QuantConnect.Orders;
+using QuantConnect.Algorithm;
+using QuantConnect.DataSource;
+using System.Collections.Generic;
+using System;
+
+namespace QuantConnect.DataLibrary.Tests
+{
+    public class Demonstration : QCAlgorithm
+    {
+        private decimal? _greatestValue = null;
+        private Dictionary<string, int> _numberOfPoints = new()
+        {
+            { "JPINTDDMEJPY", 180 },
+            { "TRINTDEXR", 180 },
+            { "CBETHUSD", 179 },
+            { "VXGOGCLS", 126 },
+            { "DCPN3M", 114 },
+            { "BAMLEMPTPRVICRPITRIV", 129 }
+        };
+
+        private Dictionary<string, Symbol> _symbols = new Dictionary<string, Symbol>();
+
+        public override void Initialize()
+        {
+            SetStartDate(2021, 1, 1);  //Set Start Date
+            SetEndDate(2021, 7, 1);    //Set End Date
+            AddEquity("SPY", Resolution.Daily);
+
+            Fred.SetAuthCode("your_authentication_code");
+            foreach (var ticker in _numberOfPoints.Keys)
+            {
+                _symbols[ticker] = AddData<Fred>(ticker).Symbol;
+
+                //Historical data
+                var historicalDataCount = (History<Fred>(_symbols[ticker], 180, Resolution.Daily)).Count();
+                if (historicalDataCount < _numberOfPoints[ticker])
+                {
+                    throw new Exception($"We expected more than {_numberOfPoints[ticker]} points for {ticker} FRED symbol, but just {historicalDataCount} points were obtained.");
+                }
+
+                Debug($"We got {historicalDataCount} items from our history request for {ticker} FRED data");
+            }
+        }
+
+        public override void OnData(Slice slice)
+        {
+            var data = slice.Get<Fred>();
+
+            if (!data.IsNullOrEmpty() && !Portfolio.Invested)
+            {
+                Debug(data.ToString());
+                var symbolsWithData = data.Values.Count;
+                var greatestValue = data.Values.Select(x => x.Value).Max();
+                if (symbolsWithData >= 5
+                    && _greatestValue != null
+                    && greatestValue > _greatestValue)
+                {
+                    SetHoldings("SPY", 1);
+                }
+                else
+                {
+                    SetHoldings("SPY", -1);
+                }
+
+                _greatestValue = greatestValue;
+            }
+        }
+
+        public override void OnOrderEvent(OrderEvent orderEvent)
+        {
+            if (orderEvent.Status.IsFill())
+            {
+                Debug($"Purchased Stock: {orderEvent.Symbol}");
+            }
+        }
+    }
+}

Demonstration.py

@@ -0,0 +1,55 @@
+# QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+# Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from AlgorithmImports import *
+
+class Demonstration(QCAlgorithm):
+    def initialize(self):
+        self.set_start_date(2021, 1, 1)
+        self.set_end_date(2021, 7, 1)
+
+        number_of_points = {"JPINTDDMEJPY" : 180, "TRINTDEXR" : 180, "CBETHUSD" : 179, "VXGOGCLS" : 126, "DCPN3M" : 114, "BAMLEMPTPRVICRPITRIV" : 129}
+        self.symbols = {}
+
+        Fred.set_auth_code("your_authentication_code")
+        self.add_equity("SPY", Resolution.DAILY)
+        for ticker in number_of_points.keys():
+            self.symbols[ticker] = self.add_data(Fred, ticker, Resolution.DAILY).symbol
+
+            historical_data_count = len(self.history(Fred, self.symbols[ticker], 180, Resolution.DAILY))
+            if historical_data_count < number_of_points[ticker]:
+                raise Exception(f"We expected more than {number_of_points[ticker]} points for {ticker} FRED symbol, but just {historical_data_count} points were obtained.");
+
+            self.debug(f"We got {historical_data_count} items from our history request for {ticker} FRED data")
+
+        self.greatest_value = -1;
+
+    def on_data(self, data):
+        data = data.get(Fred)
+
+        if len(data.Values) != 0 and (not self.portfolio.invested):
+            self.debug(str(data))
+
+            symbols_with_data = len(data.Values)
+            greatest_value = max(list(map(lambda x: x.Value, data.Values)))
+
+            if symbols_with_data >= 5 and self.greatest_value != -1 and greatest_value > self.greatest_value:
+                self.set_holdings("SPY", 1)
+            else:
+                self.set_holdings("SPY", -1)
+
+            self.greatest_value = greatest_value
+
+    def on_order_event(self, order_event):
+        if order_event.status == 3:
+            self.debug(f"Purchased Stock: {order_event.symbol}")

Fred.cs

@@ -17,17 +17,51 @@
 using QuantConnect.Data;
 using System;
 using System.Collections.Generic;
-using System.IO;
+using Newtonsoft.Json;
+using System.Linq;
+using QuantConnect.Data.UniverseSelection;
+using QuantConnect.Configuration;
 
 namespace QuantConnect.DataSource
 {
     public partial class Fred : BaseData
     {
+        private static string _authCode = "";
+
         /// <summary>
         /// Data source ID
         /// </summary>
         public static int DataSourceId { get; } = 2010;
 
+        /// <summary>
+        /// Flag indicating whether or not the FRED auth code has been set yet
+        /// </summary>
+        public static bool IsAuthCodeSet
+        {
+            get;
+            private set;
+        }
+
+        /// <summary>
+        /// Static constructor for the <see cref="Fred"/>
+        /// </summary>
+        static Fred()
+        {
+            // Se the authentication code in FRED if it's set in Config
+            var potentialAuthenticationCode = Config.Get("fred-auth-token");
+            if (!string.IsNullOrEmpty(potentialAuthenticationCode))
+            {
+                SetAuthCode(potentialAuthenticationCode);
+            }
+        }
+
+        /// <summary>
+        /// Default <see cref="Fred"/> constructor
+        /// </summary>
+        public Fred()
+        {
+        }
+
         /// <summary>
         /// Return the URL string source of the file. This will be converted to a stream
         /// </summary>
@@ -39,13 +73,8 @@ public partial class Fred : BaseData
         /// </returns>
         public override SubscriptionDataSource GetSource(SubscriptionDataConfig config, DateTime date, bool isLiveMode)
         {
-            var localFilePath = Path.Combine(
-                Globals.DataFolder,
-                "alternative",
-                "fred",
-                $"{config.Symbol.Value.ToLowerInvariant()}.csv");
-
-            return new SubscriptionDataSource(localFilePath, SubscriptionTransportMedium.LocalFile);
+            var source = $"https://api.stlouisfed.org/fred/series/observations?file_type=json&observation_start=1998-01-01&api_key={_authCode}&series_id={config.Symbol.Value.ToLowerInvariant()}";
+            return new SubscriptionDataSource(source, SubscriptionTransportMedium.RemoteFile, FileFormat.UnfoldingCollection);
         }
 
         /// <summary>
@@ -55,19 +84,40 @@ public override SubscriptionDataSource GetSource(SubscriptionDataConfig config,
         /// <param name="line">Line of data</param>
         /// <param name="date">Date</param>
         /// <param name="isLiveMode">Is live mode</param>
-        /// <returns>New instance of USEnergy</returns>
+        /// <returns>New instance of FRED data</returns>
         public override BaseData Reader(SubscriptionDataConfig config, string line, DateTime date, bool isLiveMode)
         {
-            var csv = line.Split(',');
+            if (!IsAuthCodeSet)
+            {
+                throw new ArgumentNullException("The authentication code has not been set yet. " +
+                    "Please, set your FRED authentication code first, before requesting data.");
+            }
 
-            var parsedDate = Parse.DateTimeExact(csv[0], "yyyyMMdd");
-            return new Fred
+            var json = JsonConvert.DeserializeObject<FredApi>(line);
+            var entries = json.Observations.Where(x => x.Value != ".").Select(x =>
             {
-                Symbol = config.Symbol,
-                Value = Parse.Decimal(csv[1]),
-                Time = parsedDate,
-                EndTime = parsedDate + TimeSpan.FromDays(1)
-            };
+                return new Fred
+                {
+                    Symbol = config.Symbol,
+                    Value = x.Value.ToDecimal(),
+                    Time = x.Date,
+                    EndTime = x.Date + TimeSpan.FromDays(1)
+                };
+            });
+
+            return new BaseDataCollection(date, config.Symbol, entries);
+        }
+
+        /// <summary>
+        /// Set the FRED authentication code to request the data.
+        /// </summary>
+        /// <param name="authCode"></param>
+        public static void SetAuthCode(string authCode)
+        {
+            if (string.IsNullOrWhiteSpace(authCode)) return;
+
+            _authCode = authCode;
+            IsAuthCodeSet = true;
         }
 
         /// <summary>