形式
Plain text
投稿日時
2018-01-28 15:32
公開期間
無期限
  1. using System;
  2. using System.Collections.Generic;
  3. using System.Globalization;
  4. using System.IO;
  5. using System.Net;
  6. using System.Text.RegularExpressions;
  7. using System.Threading;
  8. using Zanetti.Data;
  9. namespace Zanetti.DataSource.Specialized
  10. {
  11. internal class KabutanSource : DailyDataSource
  12. {
  13. private readonly Object _syncObject = new object();
  14. private readonly List<int> _codes = new List<int>();
  15. private Queue<int> _codeQueue;
  16. private readonly List<int> _series = new List<int>();
  17. private readonly Queue<FetchResult> _resultQueue = new Queue<FetchResult>();
  18. private bool _terminate;
  19. private Exception _exception;
  20. private const int DaysAtOnce = 20; // 一度に取得する時系列の営業日数
  21. private const int ThreadTimes = 1;
  22. private class FetchResult
  23. {
  24. public enum Status
  25. {
  26. Success,
  27. Failure,
  28. Obsolete,
  29. Retry,
  30. }
  31. public int Code;
  32. public SortedDictionary<int, NewDailyData> Prices;
  33. public Status ReturnStatus;
  34. }
  35. public KabutanSource(int[] dates) : base(dates)
  36. {
  37. foreach (AbstractBrand brand in Env.BrandCollection.Values)
  38. {
  39. var basic = brand as BasicBrand;
  40. if (brand.Market == MarketType.B || brand.Market == MarketType.Custom ||
  41. basic == null || basic.Obsolete)
  42. continue;
  43. _codes.Add(brand.Code);
  44. }
  45. }
  46. public override int TotalStep
  47. {
  48. get { return (_codes.Count + 2) * ((_dates.Length + DaysAtOnce - 1) / DaysAtOnce); } // +2はNikkei225とTOPIX
  49. }
  50. public override void Run()
  51. {
  52. var threads = new Thread[ThreadTimes];
  53. for (var i = 0; i < threads.Length; i++)
  54. (threads[i] = new Thread(RunFetchPrices) { Name = "Fetch Thread " + i }).Start();
  55. var dates = new List<int>(_dates);
  56. try
  57. {
  58. do
  59. {
  60. // 日経平均の時系列データの存在を確認する。
  61. var n = Math.Min(DaysAtOnce, dates.Count);
  62. var original = dates.GetRange(0, n);
  63. var nikkei225 = FetchPrices((int)BuiltInIndex.Nikkei225, original);
  64. if (nikkei225.ReturnStatus != FetchResult.Status.Success)
  65. throw new Exception(string.Format("株価の取得に失敗しました。時間を置いて再試行してください。: {0}~{1}",
  66. original[0], original[original.Count - 1]));
  67. dates.RemoveRange(0, n);
  68. _series.Clear();
  69. foreach (var date in original)
  70. {
  71. if (nikkei225.Prices[date].close == 0)
  72. nikkei225.Prices.Remove(date);
  73. else
  74. _series.Add(date);
  75. }
  76. if (_series.Count == 0)
  77. return;
  78. UpdateDataFarm((int)BuiltInIndex.Nikkei225, nikkei225.Prices);
  79. SendMessage(AsyncConst.WM_ASYNCPROCESS, (int)BuiltInIndex.Nikkei225, AsyncConst.LPARAM_PROGRESS_SUCCESSFUL);
  80. _codeQueue = new Queue<int>(_codes);
  81. _codeQueue.Enqueue((int)BuiltInIndex.TOPIX);
  82. _codeQueue.Enqueue((int)BuiltInIndex.JASDAQ);
  83. //_codeQueue.Enqueue((int)BuiltInIndex.TOSHO2BU);
  84. //_codeQueue.Enqueue((int)BuiltInIndex.MOTHERS);
  85. //_codeQueue.Enqueue((int)BuiltInIndex.JPYUSD);
  86. //_codeQueue.Enqueue((int)BuiltInIndex.JPYEUR);
  87. var retry = 0;
  88. while (true)
  89. {
  90. int numCodes;
  91. lock (_syncObject)
  92. {
  93. numCodes = _codeQueue.Count;
  94. Monitor.PulseAll(_syncObject);
  95. }
  96. for (var i = 0; i < numCodes; i++)
  97. {
  98. FetchResult result;
  99. lock (_resultQueue)
  100. {
  101. while (_resultQueue.Count == 0 && _exception == null)
  102. Monitor.Wait(_resultQueue);
  103. if (_exception != null)
  104. throw _exception;
  105. result = _resultQueue.Dequeue();
  106. }
  107. switch (result.ReturnStatus)
  108. {
  109. case FetchResult.Status.Failure:
  110. case FetchResult.Status.Obsolete:
  111. continue;
  112. case FetchResult.Status.Retry:
  113. lock (_codeQueue)
  114. {
  115. _codeQueue.Enqueue(result.Code);
  116. }
  117. continue;
  118. }
  119. UpdateDataFarm(result.Code, result.Prices);
  120. SendMessage(AsyncConst.WM_ASYNCPROCESS, result.Code, AsyncConst.LPARAM_PROGRESS_SUCCESSFUL);
  121. }
  122. if (_codeQueue.Count == 0)
  123. break;
  124. if (retry++ == 10)
  125. throw new Exception(string.Format("株価の取得に失敗しました。時間を置いて再試行してください。: {0}~{1}",
  126. _series[0], _series[_series.Count - 1]));
  127. Thread.Sleep(10000);
  128. }
  129. } while (dates.Count > 0);
  130. }
  131. finally
  132. {
  133. lock (_syncObject)
  134. {
  135. _terminate = true;
  136. Monitor.PulseAll(_syncObject);
  137. }
  138. foreach (var thread in threads)
  139. thread.Join();
  140. }
  141. }
  142. public void UpdateDataFarm(int code, SortedDictionary<int, NewDailyData> prices)
  143. {
  144. var farm = (DailyDataFarm)Env.BrandCollection.FindBrand(code).CreateDailyFarm(prices.Count);
  145. var empty = farm.IsEmpty;
  146. var skip = true;
  147. foreach (var pair in prices)
  148. {
  149. if (empty && skip && pair.Value.volume == 0)
  150. continue;
  151. skip = false;
  152. farm.UpdateDataFarm(pair.Key, pair.Value);
  153. }
  154. farm.Save(Util.GetDailyDataFileName(code));
  155. }
  156. private void RunFetchPrices()
  157. {
  158. var code = 0;
  159. try
  160. {
  161. while (true)
  162. {
  163. lock (_syncObject)
  164. {
  165. while ((_codeQueue == null || _codeQueue.Count == 0) && !_terminate)
  166. Monitor.Wait(_syncObject);
  167. if (_terminate || _codeQueue == null)
  168. return;
  169. code = _codeQueue.Dequeue();
  170. }
  171. var result = FetchPrices(code, _series);
  172. lock (_resultQueue)
  173. {
  174. _resultQueue.Enqueue(result);
  175. Monitor.Pulse(_resultQueue);
  176. }
  177. }
  178. }
  179. catch (Exception e)
  180. {
  181. lock (_resultQueue)
  182. {
  183. _exception = new Exception(string.Format("{0}: {1} {2}", e.Message, code, _series[0]), e);
  184. Monitor.Pulse(_resultQueue);
  185. }
  186. }
  187. }
  188. private FetchResult FetchPrices(int code, IList<int> dates)
  189. {
  190. string page;
  191. var status = GetPage(code, Util.IntToDate(dates[0]), Util.IntToDate(dates[dates.Count - 1]), out page);
  192. if (status == FetchResult.Status.Failure || status == FetchResult.Status.Retry)
  193. return new FetchResult { Code = code, ReturnStatus = status };
  194. return ParsePage(code, page, dates);
  195. }
  196. private FetchResult.Status GetPage(int code, DateTime begin, DateTime end, out string page)
  197. {
  198. if (code == (int)BuiltInIndex.Nikkei225)
  199. code = 0;
  200. else if (code == (int)BuiltInIndex.TOPIX)
  201. code = 10;
  202. else if (code == (int)BuiltInIndex.JASDAQ)
  203. code = 102;
  204. //else if (code == (int)BuiltInIndex.TOSHO2BU)
  205. // code = 11;
  206. //else if (code == (int)BuiltInIndex.MOTHERS)
  207. // code = 12;
  208. //else if (code == (int)BuiltInIndex.JPYUSD)
  209. // code = 950;
  210. //else if (code == (int)BuiltInIndex.JPYEUR)
  211. // code = 951;
  212. var url = string.Format(
  213. "https://kabutan.jp/stock/kabuka?code={0:D4}", code);
  214. page = null;
  215. try
  216. {
  217. using (var reader = new StreamReader(Util.HttpDownload(url)))
  218. page = reader.ReadToEnd();
  219. }
  220. catch (WebException e)
  221. {
  222. switch (e.Status)
  223. {
  224. case WebExceptionStatus.ProtocolError:
  225. switch (((HttpWebResponse)e.Response).StatusCode)
  226. {
  227. case (HttpStatusCode)999:
  228. case HttpStatusCode.InternalServerError:
  229. case HttpStatusCode.BadGateway:
  230. return FetchResult.Status.Retry;
  231. }
  232. throw;
  233. case WebExceptionStatus.Timeout:
  234. case WebExceptionStatus.ConnectionClosed:
  235. case WebExceptionStatus.ReceiveFailure:
  236. case WebExceptionStatus.ConnectFailure:
  237. return FetchResult.Status.Retry;
  238. default:
  239. throw;
  240. }
  241. }
  242. Thread.Sleep(1000);
  243. return FetchResult.Status.Success;
  244. }
  245. private FetchResult ParsePage(int code, string buf, IEnumerable<int> dates)
  246. {
  247. var valid = new Regex(
  248. @"<td style=""text-align:center;"">(?<year>\d{2})/(?<month>\d?\d)/(?<day>\d?\d)</td>\r\n" +
  249. "<td>(?<open>[0-9,.]+)</td>\r\n<td>(?<high>[0-9,.]+)</td>\r\n<td>(?<low>[0-9,.]+)</td>\r\n<td>(?<close>[0-9,.]+)</td>\r\n" +
  250. "<td>.*</td>\r\n<td>.*</td>\r\n<td>(?<volume>[0-9,]+)</td>");
  251. var invalid = new Regex("該当する期間のデータはありません。<br>期間をご確認ください。");
  252. var obs = new Regex("該当する銘柄はありません。<br>再度銘柄(コード)を入力し、「表示」ボタンを押してください。");
  253. var empty = new Regex("<dl class=\"stocksInfo\">\n<dt></dt><dd class=\"category yjSb\"></dd>");
  254. if (buf == null)
  255. return null;
  256. var dict = new SortedDictionary<int, NewDailyData>();
  257. var matches = valid.Matches(buf);
  258. if (matches.Count == 0)
  259. {
  260. if (obs.Match(buf).Success || empty.Match(buf).Success) // 上場廃止(銘柄データが空のこともある)
  261. return new FetchResult { ReturnStatus = FetchResult.Status.Obsolete };
  262. if (!invalid.Match(buf).Success)
  263. throw new Exception("ページから株価を取得できません。");
  264. // ここに到達するのは出来高がないか株価が用意されていない場合
  265. }
  266. try
  267. {
  268. var shift = IsIndex(code) ? 100 : 10; // 指数は100倍、株式は10倍で記録する
  269. const NumberStyles s = NumberStyles.AllowDecimalPoint | NumberStyles.AllowThousands;
  270. foreach (Match m in matches)
  271. {
  272. var date = new DateTime(int.Parse(m.Groups["year"].Value) + 2000,
  273. int.Parse(m.Groups["month"].Value),
  274. int.Parse(m.Groups["day"].Value));
  275. dict[Util.DateToInt(date)] = new NewDailyData
  276. {
  277. open = (int)(double.Parse(m.Groups["open"].Value, s) * shift),
  278. high = (int)(double.Parse(m.Groups["high"].Value, s) * shift),
  279. low = (int)(double.Parse(m.Groups["low"].Value, s) * shift),
  280. close = (int)(double.Parse(m.Groups["close"].Value, s) * shift),
  281. volume = m.Groups["volume"].Value == "" ? 0 : (int)double.Parse(m.Groups["volume"].Value, s)
  282. };
  283. }
  284. }
  285. catch (FormatException e)
  286. {
  287. throw new Exception("ページから株価を取得できません。", e);
  288. }
  289. // 出来高がない日の株価データがないので値が0のデータを補う。
  290. foreach (var date in dates)
  291. {
  292. if (!dict.ContainsKey(date))
  293. dict[date] = new NewDailyData();
  294. }
  295. return new FetchResult { Code = code, Prices = dict, ReturnStatus = FetchResult.Status.Success };
  296. }
  297. private bool IsIndex(int code)
  298. {
  299. return code == (int)BuiltInIndex.Nikkei225 ||
  300. code == (int)BuiltInIndex.TOPIX ||
  301. code == (int)BuiltInIndex.JASDAQ;// ||
  302. //code == (int)BuiltInIndex.MOTHERS ||
  303. //code == (int)BuiltInIndex.TOSHO2BU ||
  304. //code == (int)BuiltInIndex.JPYUSD ||
  305. //code == (int)BuiltInIndex.JPYEUR;
  306. }
  307. }
  308. }
ダウンロード 印刷用表示

このコピペの URL

JavaScript での埋め込み

iframe での埋め込み

元のテキスト