Fail fast at add time naming valid markets, add subscription-free market hours lookup

Algorithm.CSharp/AddTimeValidationAndMarketHoursRegressionAlgorithm.cs

@@ -0,0 +1,146 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using QuantConnect.Interfaces;
+
+namespace QuantConnect.Algorithm.CSharp
+{
+    /// <summary>
+    /// Regression algorithm asserting that adding a security with an unknown ticker/market combination fails fast
+    /// at add time naming the markets that do have the ticker, and that <see cref="QCAlgorithm.MarketHours(Symbol)"/>
+    /// works without a subscription
+    /// </summary>
+    public class AddTimeValidationAndMarketHoursRegressionAlgorithm : QCAlgorithm, IRegressionAlgorithmDefinition
+    {
+        public override void Initialize()
+        {
+            SetStartDate(2013, 10, 07);
+            SetEndDate(2013, 10, 11);
+
+            AddEquity("SPY", Resolution.Minute);
+
+            // BNBUSD is not a coinbase pair: the add must fail naming the markets that do have it
+            AssertThrows(() => AddCrypto("BNBUSD", market: Market.Coinbase),
+                "Crypto 'BNBUSD' symbol could not be found in the database", "Markets with a 'BNBUSD' Crypto entry:", Market.Kraken);
+
+            // oanda has no crypto entries at all: the exchange hours failure must also name the valid markets
+            AssertThrows(() => AddCrypto("BTCUSD", market: Market.Oanda),
+                "Unable to locate exchange hours for Crypto-oanda-BTCUSD", "Markets with a 'BTCUSD' Crypto entry:", Market.Coinbase);
+
+            // exchange hours lookup must not require a subscription
+            var hours = MarketHours("IBM");
+            if (!TimeZones.NewYork.Equals(hours.TimeZone))
+            {
+                throw new RegressionTestException($"Unexpected time zone for IBM market hours: {hours.TimeZone}");
+            }
+
+            var cryptoHours = MarketHours(QuantConnect.Symbol.Create("BTCUSD", SecurityType.Crypto, Market.Coinbase));
+            if (!cryptoHours.IsMarketAlwaysOpen)
+            {
+                throw new RegressionTestException("Expected coinbase BTCUSD market to be always open");
+            }
+
+            // and the lookups must not have added any securities
+            if (Securities.Keys.Any(symbol => symbol.Value == "IBM" || symbol.Value == "BTCUSD" || symbol.Value == "BNBUSD"))
+            {
+                throw new RegressionTestException("No security should have been added by the market hours lookups or the failed adds");
+            }
+        }
+
+        private static void AssertThrows(Action addSecurity, params string[] expectedMessageParts)
+        {
+            try
+            {
+                addSecurity();
+            }
+            catch (ArgumentException exception)
+            {
+                foreach (var expectedMessagePart in expectedMessageParts)
+                {
+                    if (!exception.Message.Contains(expectedMessagePart, StringComparison.InvariantCulture))
+                    {
+                        throw new RegressionTestException($"Expected message to contain '{expectedMessagePart}' but was: {exception.Message}");
+                    }
+                }
+                return;
+            }
+
+            throw new RegressionTestException("Expected an ArgumentException to be thrown at add time");
+        }
+
+        /// <summary>
+        /// This is used by the regression test system to indicate if the open source Lean repository has the required data to run this algorithm.
+        /// </summary>
+        public bool CanRunLocally { get; } = true;
+
+        /// <summary>
+        /// This is used by the regression test system to indicate which languages this algorithm is written in.
+        /// </summary>
+        public List<Language> Languages { get; } = new() { Language.CSharp, Language.Python };
+
+        /// <summary>
+        /// Data Points count of all timeslices of algorithm
+        /// </summary>
+        public long DataPoints => 3943;
+
+        /// <summary>
+        /// Data Points count of the algorithm history
+        /// </summary>
+        public int AlgorithmHistoryDataPoints => 0;
+
+        /// <summary>
+        /// Final status of the algorithm
+        /// </summary>
+        public AlgorithmStatus AlgorithmStatus => AlgorithmStatus.Completed;
+
+        /// <summary>
+        /// This is used by the regression test system to indicate what the expected statistics are from running the algorithm
+        /// </summary>
+        public Dictionary<string, string> ExpectedStatistics => new Dictionary<string, string>
+        {
+            {"Total Orders", "0"},
+            {"Average Win", "0%"},
+            {"Average Loss", "0%"},
+            {"Compounding Annual Return", "0%"},
+            {"Drawdown", "0%"},
+            {"Expectancy", "0"},
+            {"Start Equity", "100000"},
+            {"End Equity", "100000"},
+            {"Net Profit", "0%"},
+            {"Sharpe Ratio", "0"},
+            {"Sortino Ratio", "0"},
+            {"Probabilistic Sharpe Ratio", "0%"},
+            {"Loss Rate", "0%"},
+            {"Win Rate", "0%"},
+            {"Profit-Loss Ratio", "0"},
+            {"Alpha", "0"},
+            {"Beta", "0"},
+            {"Annual Standard Deviation", "0"},
+            {"Annual Variance", "0"},
+            {"Information Ratio", "-8.91"},
+            {"Tracking Error", "0.223"},
+            {"Treynor Ratio", "0"},
+            {"Total Fees", "$0.00"},
+            {"Estimated Strategy Capacity", "$0"},
+            {"Lowest Capacity Asset", ""},
+            {"Portfolio Turnover", "0%"},
+            {"Drawdown Recovery", "0"},
+            {"OrderListHash", "d41d8cd98f00b204e9800998ecf8427e"}
+        };
+    }
+}

Algorithm.Python/AddTimeValidationAndMarketHoursRegressionAlgorithm.py

@@ -0,0 +1,58 @@
+# QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+# Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from AlgorithmImports import *
+
+### <summary>
+### Regression algorithm asserting that adding a security with an unknown ticker/market combination fails fast
+### at add time naming the markets that do have the ticker, and that market_hours() works without a subscription
+### </summary>
+class AddTimeValidationAndMarketHoursRegressionAlgorithm(QCAlgorithm):
+    def initialize(self):
+        self.set_start_date(2013, 10, 7)
+        self.set_end_date(2013, 10, 11)
+
+        self.add_equity("SPY", Resolution.MINUTE)
+
+        # BNBUSD is not a coinbase pair: the add must fail naming the markets that do have it
+        self.assert_throws(lambda: self.add_crypto("BNBUSD", market=Market.COINBASE),
+            ["Crypto 'BNBUSD' symbol could not be found in the database", "Markets with a 'BNBUSD' Crypto entry:", Market.KRAKEN])
+
+        # oanda has no crypto entries at all: the exchange hours failure must also name the valid markets
+        self.assert_throws(lambda: self.add_crypto("BTCUSD", market=Market.OANDA),
+            ["Unable to locate exchange hours for Crypto-oanda-BTCUSD", "Markets with a 'BTCUSD' Crypto entry:", Market.COINBASE])
+
+        # exchange hours lookup must not require a subscription
+        hours = self.market_hours("IBM")
+        if str(hours.time_zone) != "America/New_York":
+            raise AssertionError(f"Unexpected time zone for IBM market hours: {hours.time_zone}")
+
+        crypto_hours = self.market_hours(Symbol.create("BTCUSD", SecurityType.CRYPTO, Market.COINBASE))
+        if not crypto_hours.is_market_always_open:
+            raise AssertionError("Expected coinbase BTCUSD market to be always open")
+
+        # and the lookups must not have added any securities
+        if any(symbol.value in ("IBM", "BTCUSD", "BNBUSD") for symbol in self.securities.keys()):
+            raise AssertionError("No security should have been added by the market hours lookups or the failed adds")
+
+    def assert_throws(self, add_security, expected_message_parts):
+        try:
+            add_security()
+        except Exception as exception:
+            message = str(exception)
+            for expected_message_part in expected_message_parts:
+                if expected_message_part not in message:
+                    raise AssertionError(f"Expected message to contain '{expected_message_part}' but was: {message}")
+            return
+
+        raise AssertionError("Expected an exception to be thrown at add time")

Algorithm/QCAlgorithm.cs

@@ -3009,6 +3009,37 @@ public string Ticker(Symbol symbol)
             return SecurityIdentifier.Ticker(symbol, Time);
         }
 
+        /// <summary>
+        /// Gets the exchange hours of the market the given symbol trades in, from the market hours database.
+        /// The security is not required to have been added to the algorithm
+        /// </summary>
+        /// <param name="symbol">The symbol to get the exchange hours for</param>
+        /// <returns>The exchange hours of the market the given symbol trades in</returns>
+        [DocumentationAttribute(SecuritiesAndPortfolio)]
+        [DocumentationAttribute(HandlingData)]
+        public SecurityExchangeHours MarketHours(Symbol symbol)
+        {
+            return MarketHoursDatabase.GetExchangeHours(symbol.ID.Market, symbol, symbol.SecurityType);
+        }
+
+        /// <summary>
+        /// Gets the exchange hours of the market the given ticker trades in, from the market hours database.
+        /// The security is not required to have been added to the algorithm
+        /// </summary>
+        /// <param name="ticker">The ticker to get the exchange hours for. If it has not been added to the algorithm,
+        /// it is assumed to be an equity ticker in the default equity market</param>
+        /// <returns>The exchange hours of the market the given ticker trades in</returns>
+        [DocumentationAttribute(SecuritiesAndPortfolio)]
+        [DocumentationAttribute(HandlingData)]
+        public SecurityExchangeHours MarketHours(string ticker)
+        {
+            if (!SymbolCache.TryGetSymbol(ticker, out var symbol))
+            {
+                symbol = QuantConnect.Symbol.Create(ticker, SecurityType.Equity, GetMarket(null, ticker, SecurityType.Equity));
+            }
+            return MarketHours(symbol);
+        }
+
         /// <summary>
         /// Creates and adds a new <see cref="Security"/> to the algorithm
         /// </summary>

Common/Data/Auxiliary/LocalZipMapFileProvider.cs

@@ -142,7 +142,10 @@ private MapFileResolver GetMapFileResolver(AuxiliaryDataKey auxiliaryDataKey)
                 return result;
             }
 
-            throw new InvalidOperationException($"LocalZipMapFileProvider couldn't find any map files going all the way back to {endDate.ToShortDateString()} for {market}");
+            // surface the actual cause instead of provider internals: this means there is no mapping data for the market at all
+            throw new InvalidOperationException($"LocalZipMapFileProvider couldn't find any map files going all the way back to {endDate.ToShortDateString()} for {market}. " +
+                $"Map file zips are expected at '{MapFileZipHelper.GetMapFileZipFileName(market, yesterdayNewYork, auxiliaryDataKey.SecurityType)}'. " +
+                $"This usually means there is no {auxiliaryDataKey.SecurityType} data available for the '{market}' market in the data folder.");
         }
     }
 }

Common/Messages/Messages.Brokerages.cs

@@ -245,7 +245,7 @@ public static class BinanceUSBrokerageModel
             /// <summary>
             /// String message saying: The Binance.US brokerage does not currently support Margin trading
             /// </summary>
-            public static string UnsupportedAccountType = "The Binance.US brokerage does not currently support Margin trading.";
+            public static string UnsupportedAccountType = "The Binance.US brokerage does not currently support Margin trading. Only AccountType.Cash is supported.";
         }
 
         /// <summary>
@@ -433,7 +433,7 @@ public static class CoinbaseBrokerageModel
             /// <summary>
             /// String message saying: The Coinbase brokerage does not currently support Margin trading
             /// </summary>
-            public static string UnsupportedAccountType = "The Coinbase brokerage does not currently support Margin trading.";
+            public static string UnsupportedAccountType = "The Coinbase brokerage does not currently support Margin trading. Only AccountType.Cash is supported.";
 
             /// <summary>
             /// Returns a string message saying the Stop Market orders are no longer supported since the given end date

Common/Messages/Messages.Securities.cs

@@ -652,6 +652,24 @@ public static string SuggestedMarketBasedOnTicker(string market)
             {
                 return $"Suggested market based on the provided ticker 'Market.{market.ToUpperInvariant()}'.";
             }
+
+            /// <summary>
+            /// Returns a string message listing the markets that do have an entry for the given ticker and security type
+            /// </summary>
+            [MethodImpl(MethodImplOptions.AggressiveInlining)]
+            public static string MarketsWithTickerEntry(string ticker, SecurityType securityType, IEnumerable<string> markets)
+            {
+                return $"Markets with a '{ticker}' {securityType} entry: {string.Join(", ", markets)}.";
+            }
+
+            /// <summary>
+            /// Returns a string message listing the markets that have entries for the given security type
+            /// </summary>
+            [MethodImpl(MethodImplOptions.AggressiveInlining)]
+            public static string MarketsWithSecurityTypeEntries(SecurityType securityType, IEnumerable<string> markets)
+            {
+                return $"Markets with {securityType} entries: {string.Join(", ", markets)}.";
+            }
         }
 
         /// <summary>
@@ -957,11 +975,17 @@ public static class SecurityService
         {
             /// <summary>
             /// Returns a string message saying the given Symbol could not be found in the Symbol Properties Database
+            /// for the requested market, naming the markets that do have an entry for it if any
             /// </summary>
             [MethodImpl(MethodImplOptions.AggressiveInlining)]
-            public static string SymbolNotFoundInSymbolPropertiesDatabase(QuantConnect.Symbol symbol)
+            public static string SymbolNotFoundInSymbolPropertiesDatabase(QuantConnect.Symbol symbol, IReadOnlyCollection<string> availableMarkets = null)
             {
-                return $"{symbol.SecurityType} '{symbol.Value}' symbol could not be found in the database for {symbol.ID.Market} market";
+                var message = $"{symbol.SecurityType} '{symbol.Value}' symbol could not be found in the database for {symbol.ID.Market} market.";
+                if (availableMarkets?.Count > 0)
+                {
+                    message += $" {MarketHoursDatabase.MarketsWithTickerEntry(symbol.Value, symbol.SecurityType, availableMarkets)}";
+                }
+                return message;
             }
         }
 

Common/Securities/MarketHoursDatabase.cs

@@ -212,8 +212,34 @@ public virtual Entry GetEntry(string market, string symbol, SecurityType securit
 
                     throw new ArgumentException(exception);
                 }
-                // there was nothing that really matched exactly
-                throw new ArgumentException(Messages.MarketHoursDatabase.ExchangeHoursNotFound(key));
+
+                // There was nothing that really matched exactly: fail fast naming what was requested and what does exist,
+                // e.g. adding 'BTCUSD' crypto for the 'oanda' market will name the markets that do have a 'BTCUSD' crypto entry,
+                // instead of only surfacing the internal database key
+                var message = Messages.MarketHoursDatabase.ExchangeHoursNotFound(key);
+                var marketsWithTicker = !string.IsNullOrEmpty(symbol)
+                    ? SymbolPropertiesDatabase.FromDataFolder().GetMarketsForSymbol(symbol, securityType)
+                    : new List<string>();
+                if (marketsWithTicker.Count > 0)
+                {
+                    message += $" {Messages.MarketHoursDatabase.MarketsWithTickerEntry(symbol, securityType, marketsWithTicker)}";
+                }
+                else
+                {
+                    // the ticker isn't in the symbol properties database for any market, so name the markets
+                    // that have exchange hours entries for the requested security type instead
+                    var marketsWithSecurityType = Entries.Keys
+                        .Where(entryKey => entryKey.SecurityType == securityType && entryKey.Market != key.Market)
+                        .Select(entryKey => entryKey.Market)
+                        .Distinct()
+                        .OrderBy(entryMarket => entryMarket)
+                        .ToList();
+                    if (marketsWithSecurityType.Count > 0)
+                    {
+                        message += $" {Messages.MarketHoursDatabase.MarketsWithSecurityTypeEntries(securityType, marketsWithSecurityType)}";
+                    }
+                }
+                throw new ArgumentException(message);
             }
 
             return entry;