コピペ: OmegaChart DreamVisor終了対応 KenMille.cs 追加修正ポイントには//☆DreamVisor終了対応

形式
Plain text
投稿日時
2018-09-09 13:35
公開期間
無期限
  1. /*
  2. * Copyright (c) Daisuke OKAJIMA All rights reserved.
  3. *
  4. * $Id$
  5. *
  6. */
  7. using System;
  8. using System.Text;
  9. using System.IO;
  10. using System.Diagnostics;
  11. //☆DreamVisor終了対応 新規追加
  12. using System.Collections.Generic;
  13. using System.Globalization;
  14. using System.Net;
  15. using System.Text.RegularExpressions;
  16. using System.Threading;
  17. //ここまで
  18. using Zanetti.Data;
  19. namespace Zanetti.DataSource.Specialized
  20. {
  21. internal class KenMilleDataSource : FullDataSource {
  22. public KenMilleDataSource(CodeEnumerator ce) : base(ce) {
  23. }
  24. public override byte[] DownloadFullData(BasicBrand br) {
  25. MemoryStream s = null;
  26. try {
  27. int code = br.Code;
  28. s = Util.HttpDownload(FormatDailyTimeSeriesURL(code, br.Market));
  29. s.Close();
  30. return s.ToArray();
  31. }
  32. finally {
  33. if(s!=null) s.Close();
  34. }
  35. }
  36. public override void ImportFullData(BasicBrand br, byte[] buf) {
  37. //Debug.WriteLine("DL START");
  38. FileStream d = null;
  39. int code = br.Code;
  40. string filename = Util.GetDailyDataFileName(code);
  41. bool success = false;
  42. try {
  43. d = new FileStream(filename, FileMode.Create);
  44. int records = buf[0] + (buf[1]*256); //最初の2バイトでレコード数を示している
  45. if(records > 10000) throw new FormatException("レコード数が異常です");
  46. int offset = 4;
  47. bool body_found = false;
  48. if(!VolumeIsAvailable(code)) body_found = true; //出来高データがないとわかっている奴は最初からtrue
  49. if(code==(int)BuiltInIndex.Nikkei225 || code==(int)BuiltInIndex.TOPIX) body_found = true; //日経平均、TOPIXは過去データに出来高0のゾーンがある
  50. for(int i=0; i<records; i++) {
  51. //2792など、先頭に空データの入っているいやらしい銘柄が存在する
  52. if(!body_found) {
  53. body_found = buf[offset+24]!=0; //出来高をみる
  54. //最初に見つかった位置で不正な日付データなら警告
  55. TestDateFormat(buf, offset);
  56. }
  57. if(body_found) {
  58. Debug.Assert(offset + 36 <= buf.Length);
  59. d.Write(buf, offset+0, 20);
  60. d.Write(buf, offset+24, 12);
  61. }
  62. offset += 36;
  63. }
  64. success = true;
  65. }
  66. finally {
  67. if(d!=null) d.Close();
  68. //Debug.WriteLine("DL END");
  69. if(!success && File.Exists(filename)) File.Delete(filename);
  70. }
  71. }
  72. private static string FormatDailyTimeSeriesURL(int code, MarketType mt) {
  73. if(mt==MarketType.B) code = MarketTypeToFileName((BuiltInIndex)code);
  74. string t = code<1000? "0"+(code.ToString()) : code.ToString();
  75. string post = "";
  76. if(mt==MarketType.O1 || mt==MarketType.O2) post = "o";
  77. else if(mt==MarketType.H) post = "n";
  78. else if(mt==MarketType.M) post = "z";
  79. /*
  80. * 古い仕様
  81. return String.Format("http://www.miller.co.jp/member/chart/D{0}/D{1}{2}", t[0], t, post);
  82. */
  83. //ガードが施されたあとの仕様 ガードの意図が感じられるというだけで、実際はJavaアプレットの中をちょっと覗けばOK
  84. return String.Format("http://www.miller.co.jp/kmp00/visitor/apps/cgi-bin/cv0cgt10c.cgi?type=1&q={0}{1}&dw=1&password={2}", t, post, FormatKey(code));
  85. }
  86. private static string FormatKey(int code) {
  87. StringBuilder b = new StringBuilder();
  88. if(code < 1000) code += 1000;
  89. //string s2 = ((code + 3352) * 23 + 0x11ae5).ToString();
  90. string s2 = ((code + Env.Options.KenmilleKey) * 23 + 0x100d9).ToString();
  91. b.Append(s2.Substring(2, 1));
  92. b.Append("Xe");
  93. b.Append(s2.Substring(2, 1));
  94. b.Append("e+");
  95. b.Append(s2.Substring(1, 2));
  96. b.Append("vw");
  97. b.Append(s2.Substring(3, 2));
  98. b.Append("f");
  99. b.Append(s2.Substring(2, 1));
  100. b.Append("3a");
  101. return b.ToString();
  102. }
  103. private static int MarketTypeToFileName(BuiltInIndex m) {
  104. switch(m) {
  105. case BuiltInIndex.Nikkei225:
  106. return 100;
  107. case BuiltInIndex.TOPIX:
  108. return 105;
  109. case BuiltInIndex.JASDAQ:
  110. return 115;
  111. case BuiltInIndex.Nikkei225_F:
  112. return 101;
  113. case BuiltInIndex.TOPIX_F:
  114. return 106;
  115. case BuiltInIndex.JPYUSD:
  116. return 500;
  117. case BuiltInIndex.JPYEUR:
  118. return 501;
  119. case BuiltInIndex.Dow:
  120. return 200;
  121. case BuiltInIndex.Nasdaq:
  122. return 202;
  123. case BuiltInIndex.SP500:
  124. return 201;
  125. default:
  126. throw new ArgumentException("unexpected index " + m);
  127. }
  128. }
  129. private static bool VolumeIsAvailable(int code) {
  130. //現在、通貨関係は出来高データなし
  131. return !(200<=code && code<400);
  132. }
  133. private static void TestDateFormat(byte[] buf, int offset) {
  134. unsafe {
  135. fixed(byte* p = &buf[offset]) {
  136. int t = *(int*)p;
  137. if(t==0) return; //0が入っているのはたまにある。正しい。
  138. if(t < 19760101 || t > 21000101) throw new FormatException("日付フォーマットが不正です。");
  139. }
  140. }
  141. }
  142. }
  143. //☆DreamVisor終了対応 新規追加
  144. internal class KenMilleDataVariousSource : DailyDataNoDatesSource
  145. {
  146. private readonly Object _syncObject = new object();
  147. private readonly List<int> _codes = new List<int>();
  148. private Queue<int> _codeQueue;
  149. private readonly Queue<FetchResult> _resultQueue = new Queue<FetchResult>();
  150. private bool _terminate;
  151. private Exception _exception;
  152. private const int DaysAtOnce = 5; //ダウンロード時間用なのでテキトー
  153. protected AbstractBrand _targetBrand;
  154. private class FetchResult
  155. {
  156. public enum Status
  157. {
  158. Success,
  159. Failure,
  160. Obsolete,
  161. Retry,
  162. }
  163. public int Code;
  164. public SortedDictionary<int, NewDailyData> Prices;
  165. public Status ReturnStatus;
  166. }
  167. public KenMilleDataVariousSource(CodeEnumerator ce) : base(ce)
  168. {
  169. _targetBrand = ce.Next;
  170. while (_targetBrand != null)
  171. {
  172. if (_targetBrand is BasicBrand)
  173. {
  174. try
  175. {
  176. _codes.Add(_targetBrand.Code);
  177. }
  178. catch (Exception ex)
  179. {
  180. Console.WriteLine("Download failed code " + _targetBrand.Code);
  181. Util.SilentReportCriticalError(ex);
  182. _errorMessage = ex.Message;
  183. SendMessage(AsyncConst.WM_ASYNCPROCESS, _targetBrand.Code, AsyncConst.LPARAM_PROGRESS_FAILURE);
  184. }
  185. }
  186. _targetBrand = ce.Next;
  187. }
  188. }
  189. public override int TotalStep
  190. {
  191. get { return 3; }//何レコード取るかはユーザーの更新状況に因るので3はテキトー
  192. }
  193. public override void Run()
  194. {
  195. var threads = new Thread[1];
  196. for (var i = 0; i < threads.Length; i++)
  197. (threads[i] = new Thread(RunFetchPrices) { Name = "Fetch Thread " + i }).Start();
  198. _codeQueue = new Queue<int>(_codes);
  199. try
  200. {
  201. var retry = 0;
  202. while (true)
  203. {
  204. int numCodes;
  205. lock (_syncObject)
  206. {
  207. numCodes = _codeQueue.Count;
  208. Monitor.PulseAll(_syncObject);
  209. }
  210. for (var i = 0; i < numCodes; i++)
  211. {
  212. FetchResult result;
  213. lock (_resultQueue)
  214. {
  215. while (_resultQueue.Count == 0 && _exception == null)
  216. Monitor.Wait(_resultQueue);
  217. if (_exception != null)
  218. throw _exception;
  219. result = _resultQueue.Dequeue();
  220. }
  221. switch (result.ReturnStatus)
  222. {
  223. case FetchResult.Status.Failure:
  224. case FetchResult.Status.Obsolete:
  225. continue;
  226. case FetchResult.Status.Retry:
  227. lock (_codeQueue)
  228. {
  229. _codeQueue.Enqueue(result.Code);
  230. }
  231. continue;
  232. }
  233. UpdateDataFarm(result.Code, result.Prices);
  234. SendMessage(AsyncConst.WM_ASYNCPROCESS, result.Code, AsyncConst.LPARAM_PROGRESS_SUCCESSFUL);
  235. }
  236. if (_codeQueue.Count == 0)
  237. break;
  238. if (retry++ == 10)
  239. throw new Exception(string.Format("株価の取得に失敗しました。時間を置いて再試行してください。"));
  240. Thread.Sleep(10000);
  241. }
  242. }
  243. catch (Exception ex)
  244. {
  245. Console.WriteLine(ex.Message);
  246. }
  247. finally
  248. {
  249. lock (_syncObject)
  250. {
  251. _terminate = true;
  252. Monitor.PulseAll(_syncObject);
  253. }
  254. foreach (var thread in threads)
  255. thread.Join();
  256. }
  257. }
  258. public void UpdateDataFarm(int code, SortedDictionary<int, NewDailyData> prices)
  259. {
  260. var farm = (DailyDataFarm)Env.BrandCollection.FindBrand(code).CreateDailyFarm(prices.Count);
  261. var empty = farm.IsEmpty;
  262. var skip = true;
  263. foreach (var pair in prices)
  264. {
  265. if (empty && skip && pair.Value.volume == 0)
  266. continue;
  267. skip = false;
  268. farm.UpdateDataFarm(pair.Key, pair.Value);
  269. }
  270. farm.Save(Util.GetDailyDataFileName(code));
  271. }
  272. private void RunFetchPrices()
  273. {
  274. var code = 0;
  275. try
  276. {
  277. while (true)
  278. {
  279. lock (_syncObject)
  280. {
  281. while ((_codeQueue == null || _codeQueue.Count == 0) && !_terminate)
  282. Monitor.Wait(_syncObject);
  283. if (_terminate || _codeQueue == null)
  284. return;
  285. code = _codeQueue.Dequeue();
  286. }
  287. var result = FetchPrices(code);
  288. lock (_resultQueue)
  289. {
  290. _resultQueue.Enqueue(result);
  291. Monitor.Pulse(_resultQueue);
  292. }
  293. }
  294. }
  295. catch (Exception e)
  296. {
  297. lock (_resultQueue)
  298. {
  299. _exception = new Exception(string.Format("{0}: {1}", e.Message, code), e);
  300. Monitor.Pulse(_resultQueue);
  301. }
  302. }
  303. }
  304. private FetchResult FetchPrices(int code)
  305. {
  306. string page;
  307. var status = GetPage(code, out page);
  308. if (status == FetchResult.Status.Failure || status == FetchResult.Status.Retry)
  309. return new FetchResult { Code = code, ReturnStatus = status };
  310. return ParsePage(code, page);
  311. }
  312. //ケンミレのジャスダックは有料サイトにしかない
  313. private string CodeString(int code)
  314. {
  315. var codestring = string.Empty;
  316. switch (code)
  317. {
  318. case (int)BuiltInIndex.Nikkei225_F:
  319. codestring = "0101I";
  320. break;
  321. case (int)BuiltInIndex.TOPIX_F:
  322. codestring = "0106I";
  323. break;
  324. //case (int)BuiltInIndex.Mothers:
  325. // codestring = "0130I";
  326. // break;
  327. default:
  328. codestring = string.Empty;
  329. break;
  330. }
  331. return codestring;
  332. }
  333. private FetchResult.Status GetPage(int code, out string page)
  334. {
  335. var url = string.Format(
  336. "http://miller.co.jp/kmp00/visitor/apps/cgi-bin/c00cht06.cgi?{0}", CodeString(code));
  337. page = null;
  338. try
  339. {
  340. using (var reader = new StreamReader(Util.HttpDownload(url), System.Text.Encoding.GetEncoding("shift-jis")))
  341. page = reader.ReadToEnd();
  342. }
  343. catch (WebException e)
  344. {
  345. switch (e.Status)
  346. {
  347. case WebExceptionStatus.ProtocolError:
  348. switch (((HttpWebResponse)e.Response).StatusCode)
  349. {
  350. case (HttpStatusCode)999:
  351. case HttpStatusCode.InternalServerError:
  352. case HttpStatusCode.BadGateway:
  353. return FetchResult.Status.Retry;
  354. }
  355. throw;
  356. case WebExceptionStatus.Timeout:
  357. case WebExceptionStatus.ConnectionClosed:
  358. case WebExceptionStatus.ReceiveFailure:
  359. case WebExceptionStatus.ConnectFailure:
  360. return FetchResult.Status.Retry;
  361. default:
  362. throw;
  363. }
  364. }
  365. return FetchResult.Status.Success;
  366. }
  367. private FetchResult ParsePage(int code, string buf)
  368. {
  369. var valid = new Regex(
  370. @"<td align=right nowrap>(?<year>\d{4})/(?<month>\d?\d)/(?<day>\d?\d)+</td>" +
  371. "<td align=right nowrap>(?<open>[0-9,.]+)</td>" +
  372. "<td align=right nowrap>(?<high>[0-9,.]+)</td>" +
  373. "<td align=right nowrap>(?<low>[0-9,.]+)</td>" +
  374. "<td align=right nowrap class=\"\">(?<close>[0-9,.]+)</td>" +
  375. "<td align=right nowrap>(<font color=\"#[0-9A-Za-z]+\">[-+.,0-9]+|0)</font></td>" +
  376. "<td align=right nowrap>(?<volume>[0-9,.]+)</td>"
  377. );
  378. var invalid = new Regex("該当する期間のデータはありません。<br>期間をご確認ください。");
  379. var obs = new Regex("該当する銘柄はありません。<br>再度銘柄(コード)を入力し、「表示」ボタンを押してください。");
  380. var empty = new Regex("<dl class=\"stocksInfo\">\n<dt></dt><dd class=\"category yjSb\"></dd>");
  381. if (buf == null)
  382. return null;
  383. var dict = new SortedDictionary<int, NewDailyData>();
  384. var matches = valid.Matches(buf.Replace("\r\n", "").Replace("\n", "").Replace("\t", "").Replace("bg-high", "").Replace("bg-low", "").Replace("(", "").Replace(")", "").Replace("月", "").Replace("火", "").Replace("水", "").Replace("木", "").Replace("金", "").Replace("土", "").Replace("日", ""));
  385. if (matches.Count == 0)//存在しなかったらスルーするだけ
  386. {
  387. if (obs.Match(buf).Success || empty.Match(buf).Success) // 上場廃止(銘柄データが空のこともある)
  388. return new FetchResult { ReturnStatus = FetchResult.Status.Obsolete };
  389. //else if (!invalid.Match(buf).Success)
  390. // throw new Exception("ページから株価を取得できません。");
  391. else
  392. return new FetchResult { Code = code, Prices = dict, ReturnStatus = FetchResult.Status.Success };
  393. }
  394. try
  395. {
  396. var shift = code == 151 ? 1 : IsIndex(code) ? 100 : 10; // 指数は100倍、株式は10倍で記録する
  397. const NumberStyles s = NumberStyles.AllowDecimalPoint | NumberStyles.AllowThousands;
  398. foreach (Match m in matches)
  399. {
  400. var date = new DateTime(int.Parse(m.Groups["year"].Value),
  401. int.Parse(m.Groups["month"].Value),
  402. int.Parse(m.Groups["day"].Value));
  403. dict[Util.DateToInt(date)] = new NewDailyData
  404. {
  405. open = (int)(double.Parse(m.Groups["open"].Value, s) * shift),
  406. high = (int)(double.Parse(m.Groups["high"].Value, s) * shift),
  407. low = (int)(double.Parse(m.Groups["low"].Value, s) * shift),
  408. close = (int)(double.Parse(m.Groups["close"].Value, s) * shift),
  409. volume = (int)(double.Parse(m.Groups["volume"].Value.Replace(".", ""), s))
  410. };
  411. }
  412. }
  413. catch (FormatException e)
  414. {
  415. throw new Exception("ページから株価を取得できません。", e);
  416. }
  417. return new FetchResult { Code = code, Prices = dict, ReturnStatus = FetchResult.Status.Success };
  418. }
  419. private bool IsIndex(int code)
  420. {
  421. return code == (int)BuiltInIndex.Nikkei225 ||
  422. code == (int)BuiltInIndex.TOPIX;
  423. }
  424. }
  425. }
ダウンロード 印刷用表示

このコピペの URL

JavaScript での埋め込み

iframe での埋め込み

元のテキスト