形式
Plain text
投稿日時
2018-03-17 19:11
公開期間
無期限
  1. using System;
  2. using System.Collections.Generic;
  3. using System.Globalization;
  4. using System.IO;
  5. using System.Net;
  6. using System.Text.RegularExpressions;
  7. using System.Threading;
  8. using Zanetti.Data;
  9. namespace Zanetti.DataSource.Specialized
  10. {
  11. internal class KabutanSource : DailyDataSource
  12. {
  13. private readonly Object _syncObject = new object();
  14. private readonly List<int> _codes = new List<int>();
  15. private Queue<int> _codeQueue;
  16. private readonly List<int> _series = new List<int>();
  17. private readonly Queue<FetchResult> _resultQueue = new Queue<FetchResult>();
  18. private bool _terminate;
  19. private Exception _exception;
  20. private const int DaysAtOnce = 20; // 一度に取得する時系列の営業日数
  21. private const int ThreadTimes = 1;
  22. private class FetchResult
  23. {
  24. public enum Status
  25. {
  26. Success,
  27. Failure,
  28. Obsolete,
  29. Retry,
  30. }
  31. public int Code;
  32. public SortedDictionary<int, NewDailyData> Prices;
  33. public Status ReturnStatus;
  34. }
  35. public KabutanSource(int[] dates) : base(dates)
  36. {
  37. foreach (AbstractBrand brand in Env.BrandCollection.Values)
  38. {
  39. var basic = brand as BasicBrand;
  40. if (brand.Market == MarketType.B || brand.Market == MarketType.Custom ||
  41. basic == null || basic.Obsolete)
  42. continue;
  43. _codes.Add(brand.Code);
  44. }
  45. }
  46. public override int TotalStep
  47. {
  48. get { return (_codes.Count + 2) * ((_dates.Length + DaysAtOnce - 1) / DaysAtOnce); } // +2はNikkei225とTOPIX
  49. }
  50. public override void Run()
  51. {
  52. var threads = new Thread[ThreadTimes];
  53. for (var i = 0; i < threads.Length; i++)
  54. (threads[i] = new Thread(RunFetchPrices) { Name = "Fetch Thread " + i }).Start();
  55. var dates = new List<int>(_dates);
  56. try
  57. {
  58. do
  59. {
  60. // 日経平均の時系列データの存在を確認する。
  61. var n = Math.Min(DaysAtOnce, dates.Count);
  62. var original = dates.GetRange(0, n);
  63. var nikkei225 = FetchPrices((int)BuiltInIndex.Nikkei225, original);
  64. if (nikkei225.ReturnStatus != FetchResult.Status.Success)
  65. throw new Exception(string.Format("株価の取得に失敗しました。時間を置いて再試行してください。: {0}~{1}",
  66. original[0], original[original.Count - 1]));
  67. dates.RemoveRange(0, n);
  68. _series.Clear();
  69. foreach (var date in original)
  70. {
  71. if (nikkei225.Prices[date].close == 0)
  72. nikkei225.Prices.Remove(date);
  73. else
  74. _series.Add(date);
  75. }
  76. if (_series.Count == 0)
  77. return;
  78. UpdateDataFarm((int)BuiltInIndex.Nikkei225, _dates, nikkei225.Prices);
  79. SendMessage(AsyncConst.WM_ASYNCPROCESS, (int)BuiltInIndex.Nikkei225, AsyncConst.LPARAM_PROGRESS_SUCCESSFUL);
  80. _codeQueue = new Queue<int>(_codes);
  81. _codeQueue.Enqueue((int)BuiltInIndex.TOPIX);
  82. _codeQueue.Enqueue((int)BuiltInIndex.JASDAQ);
  83. //_codeQueue.Enqueue((int)BuiltInIndex.TOSHO2BU);
  84. //_codeQueue.Enqueue((int)BuiltInIndex.MOTHERS);
  85. //_codeQueue.Enqueue((int)BuiltInIndex.JPYUSD);
  86. //_codeQueue.Enqueue((int)BuiltInIndex.JPYEUR);
  87. var retry = 0;
  88. while (true)
  89. {
  90. int numCodes;
  91. lock (_syncObject)
  92. {
  93. numCodes = _codeQueue.Count;
  94. Monitor.PulseAll(_syncObject);
  95. }
  96. for (var i = 0; i < numCodes; i++)
  97. {
  98. FetchResult result;
  99. lock (_resultQueue)
  100. {
  101. while (_resultQueue.Count == 0 && _exception == null)
  102. Monitor.Wait(_resultQueue);
  103. if (_exception != null)
  104. throw _exception;
  105. result = _resultQueue.Dequeue();
  106. }
  107. switch (result.ReturnStatus)
  108. {
  109. case FetchResult.Status.Failure:
  110. case FetchResult.Status.Obsolete:
  111. continue;
  112. case FetchResult.Status.Retry:
  113. lock (_codeQueue)
  114. {
  115. _codeQueue.Enqueue(result.Code);
  116. }
  117. continue;
  118. }
  119. UpdateDataFarm(result.Code, _dates, result.Prices);
  120. SendMessage(AsyncConst.WM_ASYNCPROCESS, result.Code, AsyncConst.LPARAM_PROGRESS_SUCCESSFUL);
  121. }
  122. if (_codeQueue.Count == 0)
  123. break;
  124. if (retry++ == 10)
  125. throw new Exception(string.Format("株価の取得に失敗しました。時間を置いて再試行してください。: {0}~{1}",
  126. _series[0], _series[_series.Count - 1]));
  127. Thread.Sleep(10000);
  128. }
  129. } while (dates.Count > 0);
  130. }
  131. finally
  132. {
  133. lock (_syncObject)
  134. {
  135. _terminate = true;
  136. Monitor.PulseAll(_syncObject);
  137. }
  138. foreach (var thread in threads)
  139. thread.Join();
  140. }
  141. }
  142. public void UpdateDataFarm(int code, int[] _dates, SortedDictionary<int, NewDailyData> prices)
  143. {
  144. var farm = (DailyDataFarm)Env.BrandCollection.FindBrand(code).CreateDailyFarm(prices.Count);
  145. var empty = farm.IsEmpty;
  146. var skip = true;
  147. foreach (var pair in prices)
  148. {
  149. if (!ChkDates(_dates, pair.Key)) continue;//差し替え その1で追加
  150. if (empty && skip && pair.Value.volume == 0)
  151. continue;
  152. skip = false;
  153. farm.UpdateDataFarm(pair.Key, pair.Value);
  154. }
  155. farm.Save(Util.GetDailyDataFileName(code));
  156. }
  157. private bool ChkDates(int[] _d, int date)//差し替え その1で追加
  158. {
  159. for(int i = 0;i<_d.Length;i++)
  160. {
  161. if (_d[i] == date)
  162. return true;
  163. }
  164. return false;
  165. }
  166. private void RunFetchPrices()
  167. {
  168. var code = 0;
  169. try
  170. {
  171. while (true)
  172. {
  173. lock (_syncObject)
  174. {
  175. while ((_codeQueue == null || _codeQueue.Count == 0) && !_terminate)
  176. Monitor.Wait(_syncObject);
  177. if (_terminate || _codeQueue == null)
  178. return;
  179. code = _codeQueue.Dequeue();
  180. }
  181. var result = FetchPrices(code, _series);
  182. lock (_resultQueue)
  183. {
  184. _resultQueue.Enqueue(result);
  185. Monitor.Pulse(_resultQueue);
  186. }
  187. }
  188. }
  189. catch (Exception e)
  190. {
  191. lock (_resultQueue)
  192. {
  193. _exception = new Exception(string.Format("{0}: {1} {2}", e.Message, code, _series[0]), e);
  194. Monitor.Pulse(_resultQueue);
  195. }
  196. }
  197. }
  198. private FetchResult FetchPrices(int code, IList<int> dates)
  199. {
  200. string page;
  201. var status = GetPage(code, Util.IntToDate(dates[0]), Util.IntToDate(dates[dates.Count - 1]), out page);
  202. if (status == FetchResult.Status.Failure || status == FetchResult.Status.Retry)
  203. return new FetchResult { Code = code, ReturnStatus = status };
  204. return ParsePage(code, page, dates);
  205. }
  206. private FetchResult.Status GetPage(int code, DateTime begin, DateTime end, out string page)
  207. {
  208. if (code == (int)BuiltInIndex.Nikkei225)
  209. code = 0;
  210. else if (code == (int)BuiltInIndex.TOPIX)
  211. code = 10;
  212. else if (code == (int)BuiltInIndex.JASDAQ)
  213. code = 102;
  214. //else if (code == (int)BuiltInIndex.TOSHO2BU)
  215. // code = 11;
  216. //else if (code == (int)BuiltInIndex.MOTHERS)
  217. // code = 12;
  218. //else if (code == (int)BuiltInIndex.JPYUSD)
  219. // code = 950;
  220. //else if (code == (int)BuiltInIndex.JPYEUR)
  221. // code = 951;
  222. var url = string.Format(
  223. "https://kabutan.jp/stock/kabuka?code={0:D4}", code);
  224. page = null;
  225. try
  226. {
  227. using (var reader = new StreamReader(Util.HttpDownload(url)))
  228. page = reader.ReadToEnd();
  229. }
  230. catch (WebException e)
  231. {
  232. switch (e.Status)
  233. {
  234. case WebExceptionStatus.ProtocolError:
  235. switch (((HttpWebResponse)e.Response).StatusCode)
  236. {
  237. case (HttpStatusCode)999:
  238. case HttpStatusCode.InternalServerError:
  239. case HttpStatusCode.BadGateway:
  240. return FetchResult.Status.Retry;
  241. }
  242. throw;
  243. case WebExceptionStatus.Timeout:
  244. case WebExceptionStatus.ConnectionClosed:
  245. case WebExceptionStatus.ReceiveFailure:
  246. case WebExceptionStatus.ConnectFailure:
  247. return FetchResult.Status.Retry;
  248. default:
  249. throw;
  250. }
  251. }
  252. Thread.Sleep(1000);
  253. return FetchResult.Status.Success;
  254. }
  255. private FetchResult ParsePage(int code, string buf, IEnumerable<int> dates)
  256. {
  257. buf = buf.Replace("-", "0");//差し替え その1で追加
  258. var valid = new Regex(
  259. @"<td style=""text-align:center;"">(?<year>\d{2})/(?<month>\d?\d)/(?<day>\d?\d)</td>\r\n" +
  260. "<td>(?<open>[0-9,.]+)</td>\r\n<td>(?<high>[0-9,.]+)</td>\r\n<td>(?<low>[0-9,.]+)</td>\r\n<td>(?<close>[0-9,.]+)</td>\r\n" +
  261. "<td>.*</td>\r\n<td>.*</td>\r\n<td>(?<volume>[0-9,]+)</td>");
  262. var invalid = new Regex("該当する期間のデータはありません。<br>期間をご確認ください。");
  263. var obs = new Regex("該当する銘柄はありません。<br>再度銘柄(コード)を入力し、「表示」ボタンを押してください。");
  264. var empty = new Regex("<dl class=\"stocksInfo\">\n<dt></dt><dd class=\"category yjSb\"></dd>");
  265. if (buf == null)
  266. return null;
  267. var dict = new SortedDictionary<int, NewDailyData>();
  268. var matches = valid.Matches(buf);
  269. if (matches.Count == 0)
  270. {
  271. if (obs.Match(buf).Success || empty.Match(buf).Success) // 上場廃止(銘柄データが空のこともある)
  272. return new FetchResult { ReturnStatus = FetchResult.Status.Obsolete };
  273. if (!invalid.Match(buf).Success)
  274. throw new Exception("ページから株価を取得できません。");
  275. // ここに到達するのは出来高がないか株価が用意されていない場合
  276. }
  277. try
  278. {
  279. var shift = IsIndex(code) ? 100 : 10; // 指数は100倍、株式は10倍で記録する
  280. const NumberStyles s = NumberStyles.AllowDecimalPoint | NumberStyles.AllowThousands;
  281. foreach (Match m in matches)
  282. {
  283. var date = new DateTime(int.Parse(m.Groups["year"].Value) + 2000,
  284. int.Parse(m.Groups["month"].Value),
  285. int.Parse(m.Groups["day"].Value));
  286. dict[Util.DateToInt(date)] = new NewDailyData
  287. {
  288. open = (int)(double.Parse(m.Groups["open"].Value, s) * shift),
  289. high = (int)(double.Parse(m.Groups["high"].Value, s) * shift),
  290. low = (int)(double.Parse(m.Groups["low"].Value, s) * shift),
  291. close = (int)(double.Parse(m.Groups["close"].Value, s) * shift),
  292. volume = m.Groups["volume"].Value == "" ? 0 : (int)double.Parse(m.Groups["volume"].Value, s)
  293. };
  294. }
  295. }
  296. catch (FormatException e)
  297. {
  298. throw new Exception("ページから株価を取得できません。", e);
  299. }
  300. // 出来高がない日の株価データがないので値が0のデータを補う。
  301. foreach (var date in dates)
  302. {
  303. if (!dict.ContainsKey(date))
  304. dict[date] = new NewDailyData();
  305. }
  306. return new FetchResult { Code = code, Prices = dict, ReturnStatus = FetchResult.Status.Success };
  307. }
  308. private bool IsIndex(int code)
  309. {
  310. return code == (int)BuiltInIndex.Nikkei225 ||
  311. code == (int)BuiltInIndex.TOPIX ||
  312. code == (int)BuiltInIndex.JASDAQ;// ||
  313. //code == (int)BuiltInIndex.MOTHERS ||
  314. //code == (int)BuiltInIndex.TOSHO2BU ||
  315. //code == (int)BuiltInIndex.JPYUSD ||
  316. //code == (int)BuiltInIndex.JPYEUR;
  317. }
  318. }
  319. }
ダウンロード 印刷用表示

このコピペの URL

JavaScript での埋め込み

iframe での埋め込み

元のテキスト