コピペ: OmegaChart DreamVisor終了対応 Yahoo.cs 追加修正ポイントには//☆DreamVisor終了対応

形式
Plain text
投稿日時
2018-09-09 13:36
公開期間
無期限
  1. // Copyright (c) 2014 panacoran <panacoran@users.sourceforge.jp>
  2. // This program is part of OmegaChart.
  3. // OmegaChart is licensed under the Apache License, Version 2.0.
  4. using System;
  5. using System.Collections.Generic;
  6. using System.Globalization;
  7. using System.IO;
  8. using System.Net;
  9. using System.Text.RegularExpressions;
  10. using System.Threading;
  11. using Zanetti.Data;
  12. namespace Zanetti.DataSource.Specialized
  13. {
  14. internal class YahooDataSource : DailyDataSource
  15. {
  16. private readonly Object _syncObject = new object();
  17. private readonly List<int> _codes = new List<int>();
  18. private Queue<int> _codeQueue;
  19. private readonly List<int> _series = new List<int>();
  20. private readonly Queue<FetchResult> _resultQueue = new Queue<FetchResult>();
  21. private bool _terminate;
  22. private Exception _exception;
  23. private const int DaysAtOnce = 20; // 一度に取得する時系列の営業日数
  24. private class FetchResult
  25. {
  26. public enum Status
  27. {
  28. Success,
  29. Failure,
  30. Obsolete,
  31. Retry,
  32. }
  33. public int Code;
  34. public SortedDictionary<int, NewDailyData> Prices;
  35. public Status ReturnStatus;
  36. }
  37. public YahooDataSource(int[] dates) : base(dates)
  38. {
  39. foreach (AbstractBrand brand in Env.BrandCollection.Values)
  40. {
  41. var basic = brand as BasicBrand;
  42. if (brand.Market == MarketType.B || brand.Market == MarketType.Custom ||
  43. basic == null || basic.Obsolete)
  44. continue;
  45. _codes.Add(brand.Code);
  46. }
  47. }
  48. public override int TotalStep
  49. {
  50. get { return (_codes.Count + 2) * ((_dates.Length + DaysAtOnce - 1) / DaysAtOnce); } // +2はNikkei225とTOPIX
  51. }
  52. public override void Run()
  53. {
  54. var threads = new Thread[2];
  55. for (var i = 0; i < threads.Length; i++)
  56. (threads[i] = new Thread(RunFetchPrices) {Name = "Fetch Thread " + i}).Start();
  57. var dates = new List<int>(_dates);
  58. try
  59. {
  60. do
  61. {
  62. // 日経平均の時系列データの存在を確認する。
  63. var n = Math.Min(DaysAtOnce, dates.Count);
  64. var original = dates.GetRange(0, n);
  65. var nikkei225 = FetchPrices((int)BuiltInIndex.Nikkei225, original);
  66. if (nikkei225.ReturnStatus != FetchResult.Status.Success)
  67. throw new Exception(string.Format("株価の取得に失敗しました。時間を置いて再試行してください。: {0}~{1}",
  68. original[0], original[original.Count - 1]));
  69. dates.RemoveRange(0, n);
  70. _series.Clear();
  71. foreach (var date in original)
  72. {
  73. if (nikkei225.Prices[date].close == 0)
  74. nikkei225.Prices.Remove(date);
  75. else
  76. _series.Add(date);
  77. }
  78. if (_series.Count == 0)
  79. return;
  80. UpdateDataFarm((int)BuiltInIndex.Nikkei225, nikkei225.Prices);
  81. SendMessage(AsyncConst.WM_ASYNCPROCESS, (int)BuiltInIndex.Nikkei225, AsyncConst.LPARAM_PROGRESS_SUCCESSFUL);
  82. _codeQueue = new Queue<int>(_codes);
  83. _codeQueue.Enqueue((int)BuiltInIndex.TOPIX);
  84. var retry = 0;
  85. while (true)
  86. {
  87. int numCodes;
  88. lock (_syncObject)
  89. {
  90. numCodes = _codeQueue.Count;
  91. Monitor.PulseAll(_syncObject);
  92. }
  93. for (var i = 0; i < numCodes; i++)
  94. {
  95. FetchResult result;
  96. lock (_resultQueue)
  97. {
  98. while (_resultQueue.Count == 0 && _exception == null)
  99. Monitor.Wait(_resultQueue);
  100. if (_exception != null)
  101. throw _exception;
  102. result = _resultQueue.Dequeue();
  103. }
  104. switch (result.ReturnStatus)
  105. {
  106. case FetchResult.Status.Failure:
  107. case FetchResult.Status.Obsolete:
  108. continue;
  109. case FetchResult.Status.Retry:
  110. lock (_codeQueue)
  111. {
  112. _codeQueue.Enqueue(result.Code);
  113. }
  114. continue;
  115. }
  116. UpdateDataFarm(result.Code, result.Prices);
  117. SendMessage(AsyncConst.WM_ASYNCPROCESS, result.Code, AsyncConst.LPARAM_PROGRESS_SUCCESSFUL);
  118. }
  119. if (_codeQueue.Count == 0)
  120. break;
  121. if (retry++ == 10)
  122. throw new Exception(string.Format("株価の取得に失敗しました。時間を置いて再試行してください。: {0}~{1}",
  123. _series[0], _series[_series.Count - 1]));
  124. Thread.Sleep(10000);
  125. }
  126. } while (dates.Count > 0);
  127. }
  128. finally
  129. {
  130. lock (_syncObject)
  131. {
  132. _terminate = true;
  133. Monitor.PulseAll(_syncObject);
  134. }
  135. foreach (var thread in threads)
  136. thread.Join();
  137. }
  138. }
  139. public void UpdateDataFarm(int code, SortedDictionary<int, NewDailyData> prices)
  140. {
  141. var farm = (DailyDataFarm)Env.BrandCollection.FindBrand(code).CreateDailyFarm(prices.Count);
  142. var empty = farm.IsEmpty;
  143. var skip = true;
  144. foreach (var pair in prices)
  145. {
  146. if (empty && skip && pair.Value.volume == 0)
  147. continue;
  148. skip = false;
  149. farm.UpdateDataFarm(pair.Key, pair.Value);
  150. }
  151. farm.Save(Util.GetDailyDataFileName(code));
  152. }
  153. private void RunFetchPrices()
  154. {
  155. var code = 0;
  156. try
  157. {
  158. while (true)
  159. {
  160. lock (_syncObject)
  161. {
  162. while ((_codeQueue == null || _codeQueue.Count == 0) && !_terminate)
  163. Monitor.Wait(_syncObject);
  164. if (_terminate || _codeQueue == null)
  165. return;
  166. code = _codeQueue.Dequeue();
  167. }
  168. var result = FetchPrices(code, _series);
  169. lock (_resultQueue)
  170. {
  171. _resultQueue.Enqueue(result);
  172. Monitor.Pulse(_resultQueue);
  173. }
  174. }
  175. }
  176. catch (Exception e)
  177. {
  178. lock (_resultQueue)
  179. {
  180. _exception = new Exception(string.Format("{0}: {1} {2}", e.Message, code, _series[0]), e);
  181. Monitor.Pulse(_resultQueue);
  182. }
  183. }
  184. }
  185. private FetchResult FetchPrices(int code, IList<int> dates)
  186. {
  187. string page;
  188. var status = GetPage(code, Util.IntToDate(dates[0]), Util.IntToDate(dates[dates.Count - 1]), out page);
  189. if (status == FetchResult.Status.Failure || status == FetchResult.Status.Retry)
  190. return new FetchResult { Code = code, ReturnStatus = status };
  191. return ParsePage(code, page, dates);
  192. }
  193. private FetchResult.Status GetPage(int code, DateTime begin, DateTime end, out string page)
  194. {
  195. if (code == (int)BuiltInIndex.Nikkei225)
  196. code = 998407;
  197. else if (code == (int)BuiltInIndex.TOPIX)
  198. code = 998405;
  199. var url = string.Format(
  200. "http://info.finance.yahoo.co.jp/history/?code={0}&sy={1}&sm={2}&sd={3}&ey={4}&em={5}&ed={6}&tm=d",
  201. code, begin.Year, begin.Month, begin.Day, end.Year, end.Month, end.Day);
  202. page = null;
  203. try
  204. {
  205. using (var reader = new StreamReader(Util.HttpDownload(url)))
  206. page = reader.ReadToEnd();
  207. }
  208. catch (WebException e)
  209. {
  210. switch (e.Status)
  211. {
  212. case WebExceptionStatus.ProtocolError:
  213. switch (((HttpWebResponse)e.Response).StatusCode)
  214. {
  215. case (HttpStatusCode)999:
  216. case HttpStatusCode.InternalServerError:
  217. case HttpStatusCode.BadGateway:
  218. return FetchResult.Status.Retry;
  219. }
  220. throw;
  221. case WebExceptionStatus.Timeout:
  222. case WebExceptionStatus.ConnectionClosed:
  223. case WebExceptionStatus.ReceiveFailure:
  224. case WebExceptionStatus.ConnectFailure:
  225. return FetchResult.Status.Retry;
  226. default:
  227. throw;
  228. }
  229. }
  230. return FetchResult.Status.Success;
  231. }
  232. private FetchResult ParsePage(int code, string buf, IEnumerable<int> dates)
  233. {
  234. var valid = new Regex(
  235. @"<td>(?<year>\d{4})年(?<month>1?\d)月(?<day>\d?\d)日</td>" +
  236. "<td>(?<open>[0-9,.]+)</td><td>(?<high>[0-9,.]+)</td><td>(?<low>[0-9,.]+)</td>" +
  237. "<td>(?<close>[0-9,.]+)</td>(?:<td>(?<volume>[0-9,]+)</td>)?");
  238. var invalid = new Regex("該当する期間のデータはありません。<br>期間をご確認ください。");
  239. var obs = new Regex("該当する銘柄はありません。<br>再度銘柄(コード)を入力し、「表示」ボタンを押してください。");
  240. var empty = new Regex("<dl class=\"stocksInfo\">\n<dt></dt><dd class=\"category yjSb\"></dd>");
  241. if (buf == null)
  242. return null;
  243. var dict = new SortedDictionary<int, NewDailyData>();
  244. var matches = valid.Matches(buf);
  245. if (matches.Count == 0)
  246. {
  247. if (obs.Match(buf).Success || empty.Match(buf).Success) // 上場廃止(銘柄データが空のこともある)
  248. return new FetchResult {ReturnStatus = FetchResult.Status.Obsolete};
  249. if (!invalid.Match(buf).Success)
  250. throw new Exception("ページから株価を取得できません。");
  251. // ここに到達するのは出来高がないか株価が用意されていない場合
  252. }
  253. try
  254. {
  255. var shift = IsIndex(code) ? 100 : 10; // 指数は100倍、株式は10倍で記録する
  256. const NumberStyles s = NumberStyles.AllowDecimalPoint | NumberStyles.AllowThousands;
  257. foreach (Match m in matches)
  258. {
  259. var date = new DateTime(int.Parse(m.Groups["year"].Value),
  260. int.Parse(m.Groups["month"].Value),
  261. int.Parse(m.Groups["day"].Value));
  262. dict[Util.DateToInt(date)] = new NewDailyData
  263. {
  264. open = (int)(double.Parse(m.Groups["open"].Value, s) * shift),
  265. high = (int)(double.Parse(m.Groups["high"].Value, s) * shift),
  266. low = (int)(double.Parse(m.Groups["low"].Value, s) * shift),
  267. close = (int)(double.Parse(m.Groups["close"].Value, s) * shift),
  268. volume = m.Groups["volume"].Value == "" ? 0 : (int)double.Parse(m.Groups["volume"].Value, s)
  269. };
  270. }
  271. }
  272. catch (FormatException e)
  273. {
  274. throw new Exception("ページから株価を取得できません。", e);
  275. }
  276. // 出来高がない日の株価データがないので値が0のデータを補う。
  277. foreach (var date in dates)
  278. {
  279. if (!dict.ContainsKey(date))
  280. dict[date] = new NewDailyData();
  281. }
  282. return new FetchResult {Code = code, Prices = dict, ReturnStatus = FetchResult.Status.Success};
  283. }
  284. private bool IsIndex(int code)
  285. {
  286. return code == (int)BuiltInIndex.Nikkei225 ||
  287. code == (int)BuiltInIndex.TOPIX;
  288. }
  289. }
  290. //☆DreamVisor終了対応 新規追加
  291. internal class YahooDataVariousSource : DailyDataNoDatesSource
  292. {
  293. private readonly Object _syncObject = new object();
  294. private readonly List<int> _codes = new List<int>();
  295. private Queue<int> _codeQueue;
  296. private readonly Queue<FetchResult> _resultQueue = new Queue<FetchResult>();
  297. private bool _terminate;
  298. private Exception _exception;
  299. private const int DaysAtOnce = 20; // 一度に取得する時系列の営業日数
  300. protected AbstractBrand _targetBrand;
  301. private class FetchResult
  302. {
  303. public enum Status
  304. {
  305. Success,
  306. Failure,
  307. Obsolete,
  308. Retry,
  309. }
  310. public int Code;
  311. public SortedDictionary<int, NewDailyData> Prices;
  312. public Status ReturnStatus;
  313. }
  314. public YahooDataVariousSource(CodeEnumerator ce) : base(ce)
  315. {
  316. _targetBrand = ce.Next;
  317. while (_targetBrand != null)
  318. {
  319. if (_targetBrand is BasicBrand)
  320. {
  321. try
  322. {
  323. _codes.Add(_targetBrand.Code);
  324. }
  325. catch (Exception ex)
  326. {
  327. Console.WriteLine("Download failed code " + _targetBrand.Code);
  328. Util.SilentReportCriticalError(ex);
  329. _errorMessage = ex.Message;
  330. SendMessage(AsyncConst.WM_ASYNCPROCESS, _targetBrand.Code, AsyncConst.LPARAM_PROGRESS_FAILURE);
  331. }
  332. }
  333. _targetBrand = ce.Next;
  334. }
  335. }
  336. public override int TotalStep
  337. {
  338. get { return 3; }//何レコード取るかはユーザーの更新状況に因るので3はテキトー
  339. }
  340. public override void Run()
  341. {
  342. var threads = new Thread[1];
  343. for (var i = 0; i < threads.Length; i++)
  344. (threads[i] = new Thread(RunFetchPrices) { Name = "Fetch Thread " + i }).Start();
  345. _codeQueue = new Queue<int>(_codes);
  346. try
  347. {
  348. var retry = 0;
  349. while (true)
  350. {
  351. int numCodes;
  352. lock (_syncObject)
  353. {
  354. numCodes = _codeQueue.Count;
  355. Monitor.PulseAll(_syncObject);
  356. }
  357. for (var i = 0; i < numCodes; i++)
  358. {
  359. FetchResult result;
  360. lock (_resultQueue)
  361. {
  362. while (_resultQueue.Count == 0 && _exception == null)
  363. Monitor.Wait(_resultQueue);
  364. if (_exception != null)
  365. throw _exception;
  366. result = _resultQueue.Dequeue();
  367. }
  368. switch (result.ReturnStatus)
  369. {
  370. case FetchResult.Status.Failure:
  371. case FetchResult.Status.Obsolete:
  372. continue;
  373. case FetchResult.Status.Retry:
  374. lock (_codeQueue)
  375. {
  376. _codeQueue.Enqueue(result.Code);
  377. }
  378. continue;
  379. }
  380. UpdateDataFarm(result.Code, result.Prices);
  381. SendMessage(AsyncConst.WM_ASYNCPROCESS, result.Code, AsyncConst.LPARAM_PROGRESS_SUCCESSFUL);
  382. }
  383. if (_codeQueue.Count == 0)
  384. break;
  385. if (retry++ == 10)
  386. throw new Exception(string.Format("株価の取得に失敗しました。時間を置いて再試行してください。"));
  387. Thread.Sleep(10000);
  388. }
  389. }
  390. finally
  391. {
  392. lock (_syncObject)
  393. {
  394. _terminate = true;
  395. Monitor.PulseAll(_syncObject);
  396. }
  397. foreach (var thread in threads)
  398. thread.Join();
  399. }
  400. }
  401. public void UpdateDataFarm(int code, SortedDictionary<int, NewDailyData> prices)
  402. {
  403. var farm = (DailyDataFarm)Env.BrandCollection.FindBrand(code).CreateDailyFarm(prices.Count);
  404. var empty = farm.IsEmpty;
  405. var skip = true;
  406. foreach (var pair in prices)
  407. {
  408. //ヤフーファイナンスのジャスダックは出来高なし
  409. //if (empty && skip && pair.Value.volume == 0)
  410. // continue;
  411. //skip = false;
  412. farm.UpdateDataFarm(pair.Key, pair.Value);
  413. }
  414. farm.Save(Util.GetDailyDataFileName(code));
  415. }
  416. private void RunFetchPrices()
  417. {
  418. var code = 0;
  419. try
  420. {
  421. while (true)
  422. {
  423. lock (_syncObject)
  424. {
  425. while ((_codeQueue == null || _codeQueue.Count == 0) && !_terminate)
  426. Monitor.Wait(_syncObject);
  427. if (_terminate || _codeQueue == null)
  428. return;
  429. code = _codeQueue.Dequeue();
  430. }
  431. var result = FetchPrices(code);
  432. lock (_resultQueue)
  433. {
  434. _resultQueue.Enqueue(result);
  435. Monitor.Pulse(_resultQueue);
  436. }
  437. }
  438. }
  439. catch (Exception e)
  440. {
  441. lock (_resultQueue)
  442. {
  443. _exception = new Exception(string.Format("{0}: {1}", e.Message, code), e);
  444. Monitor.Pulse(_resultQueue);
  445. }
  446. }
  447. }
  448. private FetchResult FetchPrices(int code)
  449. {
  450. string page;
  451. var status = GetPage(code, out page);
  452. if (status == FetchResult.Status.Failure || status == FetchResult.Status.Retry)
  453. return new FetchResult { Code = code, ReturnStatus = status };
  454. return ParsePage(code, page);
  455. }
  456. private string CodeString(int code)
  457. {
  458. var codestring = string.Empty;
  459. switch (code)
  460. {
  461. case (int)BuiltInIndex.Nikkei225:
  462. codestring = "998407";
  463. break;
  464. case (int)BuiltInIndex.TOPIX:
  465. codestring = "998405";
  466. break;
  467. case (int)BuiltInIndex.JASDAQ:
  468. codestring = "23337";
  469. break;
  470. case (int)BuiltInIndex.JPYUSD:
  471. codestring = "USDJPY=X";
  472. break;
  473. case (int)BuiltInIndex.JPYEUR:
  474. codestring = "EURJPY=X";
  475. break;
  476. default:
  477. codestring = string.Empty;
  478. break;
  479. }
  480. return codestring;
  481. }
  482. private FetchResult.Status GetPage(int code, out string page)
  483. {
  484. var url = string.Format(
  485. "https://stocks.finance.yahoo.co.jp/stocks/history/?code={0}", CodeString(code));
  486. page = null;
  487. try
  488. {
  489. using (var reader = new StreamReader(Util.HttpDownload(url)))
  490. page = reader.ReadToEnd();
  491. }
  492. catch (WebException e)
  493. {
  494. switch (e.Status)
  495. {
  496. case WebExceptionStatus.ProtocolError:
  497. switch (((HttpWebResponse)e.Response).StatusCode)
  498. {
  499. case (HttpStatusCode)999:
  500. case HttpStatusCode.InternalServerError:
  501. case HttpStatusCode.BadGateway:
  502. return FetchResult.Status.Retry;
  503. }
  504. throw;
  505. case WebExceptionStatus.Timeout:
  506. case WebExceptionStatus.ConnectionClosed:
  507. case WebExceptionStatus.ReceiveFailure:
  508. case WebExceptionStatus.ConnectFailure:
  509. return FetchResult.Status.Retry;
  510. default:
  511. throw;
  512. }
  513. }
  514. return FetchResult.Status.Success;
  515. }
  516. private FetchResult ParsePage(int code, string buf)
  517. {
  518. var valid = new Regex(
  519. @"<td>(?<year>\d{4})年(?<month>1?\d)月(?<day>\d?\d)日</td>" +
  520. "<td>(?<open>[0-9,.]+)</td><td>(?<high>[0-9,.]+)</td><td>(?<low>[0-9,.]+)</td>" +
  521. "<td>(?<close>[0-9,.]+)</td>");
  522. var invalid = new Regex("該当する期間のデータはありません。<br>期間をご確認ください。");
  523. var obs = new Regex("該当する銘柄はありません。<br>再度銘柄(コード)を入力し、「表示」ボタンを押してください。");
  524. var empty = new Regex("<dl class=\"stocksInfo\">\n<dt></dt><dd class=\"category yjSb\"></dd>");
  525. if (buf == null)
  526. return null;
  527. var dict = new SortedDictionary<int, NewDailyData>();
  528. var matches = valid.Matches(buf.Replace("\r\n", "").Replace("\n", ""));
  529. if (matches.Count == 0)
  530. {
  531. if (obs.Match(buf).Success || empty.Match(buf).Success) // 上場廃止(銘柄データが空のこともある)
  532. return new FetchResult { ReturnStatus = FetchResult.Status.Obsolete };
  533. if (!invalid.Match(buf).Success)
  534. throw new Exception("ページから株価を取得できません。");
  535. // ここに到達するのは出来高がないか株価が用意されていない場合
  536. }
  537. try
  538. {
  539. var shift = 100; // 為替は100倍で記録する
  540. const NumberStyles s = NumberStyles.AllowDecimalPoint | NumberStyles.AllowThousands;
  541. foreach (Match m in matches)
  542. {
  543. var date = new DateTime(int.Parse(m.Groups["year"].Value),
  544. int.Parse(m.Groups["month"].Value),
  545. int.Parse(m.Groups["day"].Value));
  546. dict[Util.DateToInt(date)] = new NewDailyData
  547. {
  548. open = (int)(double.Parse(m.Groups["open"].Value, s) * shift),
  549. high = (int)(double.Parse(m.Groups["high"].Value, s) * shift),
  550. low = (int)(double.Parse(m.Groups["low"].Value, s) * shift),
  551. close = (int)(double.Parse(m.Groups["close"].Value, s) * shift),
  552. volume = 0
  553. };
  554. }
  555. }
  556. catch (FormatException e)
  557. {
  558. throw new Exception("ページから株価を取得できません。", e);
  559. }
  560. return new FetchResult { Code = code, Prices = dict, ReturnStatus = FetchResult.Status.Success };
  561. }
  562. }
  563. //☆DreamVisor終了対応 新規追加
  564. internal class YahooDataUsSource : DailyDataNoDatesSource
  565. {
  566. private readonly Object _syncObject = new object();
  567. private readonly List<int> _codes = new List<int>();
  568. private Queue<int> _codeQueue;
  569. private readonly Queue<FetchResult> _resultQueue = new Queue<FetchResult>();
  570. private bool _terminate;
  571. private Exception _exception;
  572. private const int DaysAtOnce = 5; //ダウンロード時間用なのでテキトー
  573. protected AbstractBrand _targetBrand;
  574. private class FetchResult
  575. {
  576. public enum Status
  577. {
  578. Success,
  579. Failure,
  580. Obsolete,
  581. Retry,
  582. }
  583. public int Code;
  584. public SortedDictionary<int, NewDailyData> Prices;
  585. public Status ReturnStatus;
  586. }
  587. public YahooDataUsSource(CodeEnumerator ce) : base(ce)
  588. {
  589. _targetBrand = ce.Next;
  590. while (_targetBrand != null)
  591. {
  592. if (_targetBrand is BasicBrand)
  593. {
  594. try
  595. {
  596. _codes.Add(_targetBrand.Code);
  597. }
  598. catch (Exception ex)
  599. {
  600. Console.WriteLine("Download failed code " + _targetBrand.Code);
  601. Util.SilentReportCriticalError(ex);
  602. _errorMessage = ex.Message;
  603. SendMessage(AsyncConst.WM_ASYNCPROCESS, _targetBrand.Code, AsyncConst.LPARAM_PROGRESS_FAILURE);
  604. }
  605. }
  606. _targetBrand = ce.Next;
  607. }
  608. }
  609. public override int TotalStep
  610. {
  611. get { return 3; }//何レコード取るかはユーザーの更新状況に因るので3はテキトー
  612. }
  613. public override void Run()
  614. {
  615. var threads = new Thread[1];
  616. for (var i = 0; i < threads.Length; i++)
  617. (threads[i] = new Thread(RunFetchPrices) { Name = "Fetch Thread " + i }).Start();
  618. _codeQueue = new Queue<int>(_codes);
  619. try
  620. {
  621. var retry = 0;
  622. while (true)
  623. {
  624. int numCodes;
  625. lock (_syncObject)
  626. {
  627. numCodes = _codeQueue.Count;
  628. Monitor.PulseAll(_syncObject);
  629. }
  630. for (var i = 0; i < numCodes; i++)
  631. {
  632. FetchResult result;
  633. lock (_resultQueue)
  634. {
  635. while (_resultQueue.Count == 0 && _exception == null)
  636. Monitor.Wait(_resultQueue);
  637. if (_exception != null)
  638. throw _exception;
  639. result = _resultQueue.Dequeue();
  640. }
  641. switch (result.ReturnStatus)
  642. {
  643. case FetchResult.Status.Failure:
  644. case FetchResult.Status.Obsolete:
  645. continue;
  646. case FetchResult.Status.Retry:
  647. lock (_codeQueue)
  648. {
  649. _codeQueue.Enqueue(result.Code);
  650. }
  651. continue;
  652. }
  653. UpdateDataFarm(result.Code, result.Prices);
  654. SendMessage(AsyncConst.WM_ASYNCPROCESS, result.Code, AsyncConst.LPARAM_PROGRESS_SUCCESSFUL);
  655. }
  656. if (_codeQueue.Count == 0)
  657. break;
  658. if (retry++ == 10)
  659. throw new Exception(string.Format("株価の取得に失敗しました。時間を置いて再試行してください。"));
  660. Thread.Sleep(10000);
  661. }
  662. }
  663. finally
  664. {
  665. lock (_syncObject)
  666. {
  667. _terminate = true;
  668. Monitor.PulseAll(_syncObject);
  669. }
  670. foreach (var thread in threads)
  671. thread.Join();
  672. }
  673. }
  674. public void UpdateDataFarm(int code, SortedDictionary<int, NewDailyData> prices)
  675. {
  676. var farm = (DailyDataFarm)Env.BrandCollection.FindBrand(code).CreateDailyFarm(prices.Count);
  677. var empty = farm.IsEmpty;
  678. var skip = true;
  679. foreach (var pair in prices)
  680. {
  681. if (empty && skip && pair.Value.volume == 0)
  682. continue;
  683. skip = false;
  684. farm.UpdateDataFarm(pair.Key, pair.Value);
  685. }
  686. farm.Save(Util.GetDailyDataFileName(code));
  687. }
  688. private void RunFetchPrices()
  689. {
  690. var code = 0;
  691. try
  692. {
  693. while (true)
  694. {
  695. lock (_syncObject)
  696. {
  697. while ((_codeQueue == null || _codeQueue.Count == 0) && !_terminate)
  698. Monitor.Wait(_syncObject);
  699. if (_terminate || _codeQueue == null)
  700. return;
  701. code = _codeQueue.Dequeue();
  702. }
  703. var result = FetchPrices(code);
  704. lock (_resultQueue)
  705. {
  706. _resultQueue.Enqueue(result);
  707. Monitor.Pulse(_resultQueue);
  708. }
  709. }
  710. }
  711. catch (Exception e)
  712. {
  713. lock (_resultQueue)
  714. {
  715. _exception = new Exception(string.Format("{0}: {1}", e.Message, code), e);
  716. Monitor.Pulse(_resultQueue);
  717. }
  718. }
  719. }
  720. private FetchResult FetchPrices(int code)
  721. {
  722. string page;
  723. var status = GetPage(code, out page);
  724. if (status == FetchResult.Status.Failure || status == FetchResult.Status.Retry)
  725. return new FetchResult { Code = code, ReturnStatus = status };
  726. return ParsePage(code, page);
  727. }
  728. private string CodeString(int code)
  729. {
  730. var codestring = string.Empty;
  731. switch (code)
  732. {
  733. case (int)BuiltInIndex.Dow:
  734. codestring = "%5EDJI";
  735. break;
  736. case (int)BuiltInIndex.Nasdaq:
  737. codestring = "%5EIXIC";
  738. break;
  739. case (int)BuiltInIndex.SP500:
  740. codestring = "%5EGSPC";
  741. break;
  742. default:
  743. codestring = string.Empty;
  744. break;
  745. }
  746. return codestring;
  747. }
  748. private FetchResult.Status GetPage(int code, out string page)
  749. {
  750. var url = string.Format(
  751. "https://finance.yahoo.com/quote/{0}/history", CodeString(code));
  752. page = null;
  753. try
  754. {
  755. using (var reader = new StreamReader(Util.HttpDownload(url)))
  756. page = reader.ReadToEnd();
  757. }
  758. catch (WebException e)
  759. {
  760. switch (e.Status)
  761. {
  762. case WebExceptionStatus.ProtocolError:
  763. switch (((HttpWebResponse)e.Response).StatusCode)
  764. {
  765. case (HttpStatusCode)999:
  766. case HttpStatusCode.InternalServerError:
  767. case HttpStatusCode.BadGateway:
  768. return FetchResult.Status.Retry;
  769. }
  770. throw;
  771. case WebExceptionStatus.Timeout:
  772. case WebExceptionStatus.ConnectionClosed:
  773. case WebExceptionStatus.ReceiveFailure:
  774. case WebExceptionStatus.ConnectFailure:
  775. return FetchResult.Status.Retry;
  776. default:
  777. throw;
  778. }
  779. }
  780. return FetchResult.Status.Success;
  781. }
  782. private FetchResult ParsePage(int code, string buf)
  783. {
  784. var valid = new Regex(
  785. "<td class=\"Py\\(10px\\) Ta\\(start\\) Pend\\(10px\\)\" data-reactid=\"[0-9]+\"><span data-reactid=\"[0-9]+\">(?<month>[A-Za-z]+) (?<day>\\d?\\d), (?<year>\\d{4})</span></td>" +
  786. "<td class=\"Py\\(10px\\) Pstart\\(10px\\)\" data-reactid=\"[0-9]+\"><span data-reactid=\"[0-9]+\">(?<open>[0-9,.]+)</span></td>" +
  787. "<td class=\"Py\\(10px\\) Pstart\\(10px\\)\" data-reactid=\"[0-9]+\"><span data-reactid=\"[0-9]+\">(?<high>[0-9,.]+)</span></td>" +
  788. "<td class=\"Py\\(10px\\) Pstart\\(10px\\)\" data-reactid=\"[0-9]+\"><span data-reactid=\"[0-9]+\">(?<low>[0-9,.]+)</span></td>" +
  789. "<td class=\"Py\\(10px\\) Pstart\\(10px\\)\" data-reactid=\"[0-9]+\"><span data-reactid=\"[0-9]+\">(?<close>[0-9,.]+)</span></td>" +
  790. "<td class=\"Py\\(10px\\) Pstart\\(10px\\)\" data-reactid=\"[0-9]+\"><span data-reactid=\"[0-9]+\">(?<ajs>[0-9,.]+)</span></td>" +
  791. "<td class=\"Py\\(10px\\) Pstart\\(10px\\)\" data-reactid=\"[0-9]+\"><span data-reactid=\"[0-9]+\">(?<volume>[0-9,.]+)</span></td></tr>"
  792. );
  793. var invalid = new Regex("該当する期間のデータはありません。<br>期間をご確認ください。");
  794. var obs = new Regex("該当する銘柄はありません。<br>再度銘柄(コード)を入力し、「表示」ボタンを押してください。");
  795. var empty = new Regex("<dl class=\"stocksInfo\">\n<dt></dt><dd class=\"category yjSb\"></dd>");
  796. if (buf == null)
  797. return null;
  798. var dict = new SortedDictionary<int, NewDailyData>();
  799. var matches = valid.Matches(buf.Replace("\r\n", "").Replace("\n", ""));
  800. if (matches.Count == 0)
  801. {
  802. if (obs.Match(buf).Success || empty.Match(buf).Success) // 上場廃止(銘柄データが空のこともある)
  803. return new FetchResult { ReturnStatus = FetchResult.Status.Obsolete };
  804. if (!invalid.Match(buf).Success)
  805. throw new Exception("ページから株価を取得できません。");
  806. // ここに到達するのは出来高がないか株価が用意されていない場合
  807. }
  808. try
  809. {
  810. var shift = 100; // 指数は100倍で記録する
  811. var volume_down = 1000;//出来高は必要に応じて桁落ちさせる
  812. const NumberStyles s = NumberStyles.AllowDecimalPoint | NumberStyles.AllowThousands;
  813. foreach (Match m in matches)
  814. {
  815. var date = DateTime.Now;
  816. if (!DateTime.TryParse(string.Format("{0} {1}, {2}", m.Groups["month"].Value, m.Groups["day"].Value, m.Groups["year"].Value), out date)) continue;
  817. dict[Util.DateToInt(date)] = new NewDailyData
  818. {
  819. open = (int)(double.Parse(m.Groups["open"].Value, s) * shift),
  820. high = (int)(double.Parse(m.Groups["high"].Value, s) * shift),
  821. low = (int)(double.Parse(m.Groups["low"].Value, s) * shift),
  822. close = (int)(double.Parse(m.Groups["close"].Value, s) * shift),
  823. volume = (int)(double.Parse(m.Groups["volume"].Value, s) / volume_down)
  824. };
  825. }
  826. }
  827. catch (FormatException e)
  828. {
  829. throw new Exception("ページから株価を取得できません。", e);
  830. }
  831. return new FetchResult { Code = code, Prices = dict, ReturnStatus = FetchResult.Status.Success };
  832. }
  833. }
  834. }
ダウンロード 印刷用表示

このコピペの URL

JavaScript での埋め込み

iframe での埋め込み

元のテキスト