[GitHub] asfgit closed pull request #64: New Page for TC Bot: Long Running Tests Report
Locked
 
|
asfgit closed pull request #64: New Page for TC Bot: Long Running Tests Report
URL: https://github.com/apache/ignite-teamcity-bot/pull/64 This is a PR merged from a forked repository. As GitHub hides the original diff on merge, it is displayed below for the sake of provenance: As this is a foreign pull request (from a fork), the diff is supplied below (as it won't show otherwise due to GitHub magic): diff --git a/.gitignore b/.gitignore index 73dc8367..fa67cf10 100644 --- a/.gitignore +++ b/.gitignore @@ -14,6 +14,9 @@ .idea/ *.iml +# gradle build files +.gradle/ + # Package Files # *.jar *.war diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/chain/BuildChainProcessor.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/chain/BuildChainProcessor.java index 325fffc4..06d181d2 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/chain/BuildChainProcessor.java +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/chain/BuildChainProcessor.java @@ -19,6 +19,7 @@ import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.Comparator; import java.util.List; import java.util.Map; @@ -50,6 +51,8 @@ import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildCompacted; import org.apache.ignite.ci.util.FutureUtil; import org.apache.ignite.ci.web.TcUpdatePool; +import org.apache.ignite.ci.web.model.long_running.LRTest; +import org.apache.ignite.ci.web.model.long_running.SuiteLRTestsSummary; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import org.slf4j.Logger; @@ -68,6 +71,85 @@ /** Compactor. */ @Inject private IStringCompactor compactor; + /** + * Collects data about all long-running tests (run time more than one minute) across all suites in RunAll chain + * in master branch. + * + * @param teamcityIgnited interface to TC bot database. + * @param entryPoints + * @return list of summaries about individual suite runs. + */ + public List<SuiteLRTestsSummary> loadLongRunningTestsSummary( + ITeamcityIgnited teamcityIgnited, + Collection<BuildRef> entryPoints + ) { + final List<SuiteLRTestsSummary> res = new ArrayList<>(); + + if (entryPoints.isEmpty()) + return res; + + Map<Integer, FatBuildCompacted> builds = new ConcurrentHashMap<>(); + + final Stream<FatBuildCompacted> entryPointsFatBuilds = entryPoints.stream().map(BuildRef::getId) + .filter(Objects::nonNull) + .filter(id -> !builds.containsKey(id)) //load and propagate only new entry points + .map(id -> builds.computeIfAbsent(id, teamcityIgnited::getFatBuild)); + + final ExecutorService svc = tcUpdatePool.getService(); + + final Stream<FatBuildCompacted> depsFirstLevel = entryPointsFatBuilds + .map(ref -> svc.submit(() -> dependencies(teamcityIgnited, builds, ref))) + .collect(Collectors.toList()) + .stream() + .flatMap(fut -> FutureUtil.getResult(fut)); + + depsFirstLevel + .filter(b -> !b.isComposite() && b.getTestsCount() > 0) + .forEach(b -> + { + List<LRTest> lrTests = new ArrayList<>(); + + b.getAllTests() + .filter(t -> t.getDuration() > 60 * 1000) + .forEach( + t -> lrTests + .add(new LRTest(t.testName(compactor), t.getDuration(), null)) + ); + + if (!lrTests.isEmpty()) { + Collections.sort(lrTests, (test0, test1) -> { + long t0 = test0.time; + long t1 = test1.time; + + if (t0 < t1) + return 1; + + if (t0 == t1) + return 0; + + return -1; + }); + + res.add( + new SuiteLRTestsSummary(b.buildTypeName(compactor), + b.buildDuration(compactor) / b.getTestsCount(), + lrTests)); + } + }); + + Collections.sort(res, (s0, s1) -> { + if (s0.testAvgTime < s1.testAvgTime) + return 1; + + if (s0.testAvgTime == s1.testAvgTime) + return 0; + + return -1; + }); + + return res; + } + /** * @param teamcity Teamcity. * @param teamcityIgnited @@ -111,7 +193,7 @@ public FullChainRunCtx loadFullChainContext( .stream() .flatMap(fut -> FutureUtil.getResult(fut)); - // builds may became non unique because of rece in filtering and acquiring deps + // builds may became non unique because of race in filtering and acquiring deps final List<Future<Stream<FatBuildCompacted>>> phase3Submitted = secondLevelDeps .map((fatBuild) -> svc.submit( () -> replaceWithRecent(teamcityIgnited, includeLatestRebuild, builds, fatBuild, entryPoints.size()))) diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/chain/TrackedBranchChainsProcessor.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/chain/TrackedBranchChainsProcessor.java index 2d61c434..53852738 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/chain/TrackedBranchChainsProcessor.java +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/chain/TrackedBranchChainsProcessor.java @@ -23,18 +23,19 @@ import javax.inject.Inject; import org.apache.ignite.ci.HelperConfig; import org.apache.ignite.ci.IAnalyticsEnabledTeamcity; -import org.apache.ignite.ci.teamcity.ignited.ITeamcityIgnited; -import org.apache.ignite.ci.teamcity.ignited.ITeamcityIgnitedProvider; -import org.apache.ignite.ci.teamcity.restcached.ITcServerProvider; import org.apache.ignite.ci.analysis.FullChainRunCtx; import org.apache.ignite.ci.analysis.mode.LatestRebuildMode; import org.apache.ignite.ci.analysis.mode.ProcessLogsMode; import org.apache.ignite.ci.conf.BranchTracked; import org.apache.ignite.ci.di.AutoProfiling; import org.apache.ignite.ci.tcmodel.hist.BuildRef; +import org.apache.ignite.ci.teamcity.ignited.ITeamcityIgnited; +import org.apache.ignite.ci.teamcity.ignited.ITeamcityIgnitedProvider; +import org.apache.ignite.ci.teamcity.restcached.ITcServerProvider; import org.apache.ignite.ci.user.ICredentialsProv; import org.apache.ignite.ci.web.model.current.ChainAtServerCurrentStatus; import org.apache.ignite.ci.web.model.current.TestFailuresSummary; +import org.apache.ignite.ci.web.model.long_running.FullLRTestsSummary; import org.apache.ignite.ci.web.rest.parms.FullQueryParams; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; @@ -124,4 +125,46 @@ public TestFailuresSummary getTrackedBranchTestFailures( return res; } + + /** + * Collects data about all long-running tests (run time more than one minute) within one transfer object. + * + * @param branch + * @param creds + * @return + */ + public FullLRTestsSummary getTrackedBranchLongRunningTestsSummary(@Nullable String branch, + ICredentialsProv creds) { + FullLRTestsSummary summary = new FullLRTestsSummary(); + + final String branchNn = isNullOrEmpty(branch) ? FullQueryParams.DEFAULT_TRACKED_BRANCH_NAME : branch; + final BranchTracked tracked = HelperConfig.getTrackedBranches().getBranchMandatory(branchNn); + + tracked.chains.stream() + .filter(chainTracked -> creds.hasAccess(chainTracked.serverId)) + .map(chainTracked -> { + final String srvId = chainTracked.serverId; + + final String branchForTc = chainTracked.getBranchForRestMandatory(); + + IAnalyticsEnabledTeamcity teamcity = srvProv.server(srvId, creds); + + ITeamcityIgnited tcIgnited = tcIgnitedProv.server(srvId, creds); + + final List<BuildRef> buildsList = teamcity.getFinishedBuildsIncludeSnDepFailed( + chainTracked.getSuiteIdMandatory(), + branchForTc); + + List<BuildRef> chains = buildsList.stream() + .filter(ref -> !ref.isFakeStub()) + .sorted(Comparator.comparing(BuildRef::getId).reversed()) + .limit(1) + .filter(b -> b.getId() != null).collect(Collectors.toList()); + + return chainProc.loadLongRunningTestsSummary(tcIgnited, chains); + }) + .forEach(summary::addSuiteSummaries); + + return summary; + } } diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildCompacted.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildCompacted.java index 2426d92c..9c07e637 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildCompacted.java +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildCompacted.java @@ -355,6 +355,10 @@ public boolean isFailedToStart() { return tests.stream(); } + public int getTestsCount() { + return tests != null ? tests.size() : 0; + } + public Stream<String> getAllTestNames(IStringCompactor compactor) { return getAllTests().map(t -> t.testName(compactor)); } diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/long_running/FullLRTestsSummary.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/long_running/FullLRTestsSummary.java new file mode 100644 index 00000000..b71a77e7 --- /dev/null +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/long_running/FullLRTestsSummary.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ignite.ci.web.model.long_running; + +import java.util.ArrayList; +import java.util.List; + +/** + * + */ +@SuppressWarnings("WeakerAccess") +public class FullLRTestsSummary { + public List<SuiteLRTestsSummary> suiteSummaries = new ArrayList<>(); + + public void addSuiteSummaries(List<SuiteLRTestsSummary> summaries) { + suiteSummaries.addAll(summaries); + } +} diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/long_running/LRTest.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/long_running/LRTest.java new file mode 100644 index 00000000..e1f57e0c --- /dev/null +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/long_running/LRTest.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ignite.ci.web.model.long_running; + +import static org.apache.ignite.ci.util.TimeUtil.millisToDurationPrintable; + +/** + * + */ +public class LRTest { + public String name; + + public long time; + + public String timePrintable; + + public String webLink; + + public LRTest(String name, long time, String webLink) { + this.name = name; + this.time = time; + timePrintable = millisToDurationPrintable(time); + this.webLink = webLink; + } +} diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/long_running/SuiteLRTestsSummary.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/long_running/SuiteLRTestsSummary.java new file mode 100644 index 00000000..2ae2b0c2 --- /dev/null +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/long_running/SuiteLRTestsSummary.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ignite.ci.web.model.long_running; + +import java.util.List; + +import static org.apache.ignite.ci.util.TimeUtil.millisToDurationPrintable; + +/** + * + */ +@SuppressWarnings("WeakerAccess") +public class SuiteLRTestsSummary { + public String name; + + public long testAvgTime; + + public String testAvgTimePrintable; + + public List<LRTest> tests; + + public SuiteLRTestsSummary(String name, long testAvgTime, List<LRTest> tests) { + this.name = name; + this.testAvgTime = testAvgTime; + testAvgTimePrintable = millisToDurationPrintable(testAvgTime); + this.tests = tests; + } +} diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/long_running/BuildsLongRunningTestsReport.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/long_running/BuildsLongRunningTestsReport.java new file mode 100644 index 00000000..42d29f9a --- /dev/null +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/long_running/BuildsLongRunningTestsReport.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ignite.ci.web.rest.long_running; + +import javax.servlet.ServletContext; +import javax.servlet.http.HttpServletRequest; +import javax.ws.rs.GET; +import javax.ws.rs.Path; +import javax.ws.rs.Produces; +import javax.ws.rs.QueryParam; +import javax.ws.rs.core.Context; +import javax.ws.rs.core.MediaType; +import org.apache.ignite.ci.tcbot.chain.TrackedBranchChainsProcessor; +import org.apache.ignite.ci.user.ICredentialsProv; +import org.apache.ignite.ci.web.CtxListener; +import org.apache.ignite.ci.web.model.long_running.FullLRTestsSummary; +import org.jetbrains.annotations.Nullable; + +@Path(BuildsLongRunningTestsReport.LONG_RUNNING_SUMMARY) +@Produces(MediaType.APPLICATION_JSON) +public class BuildsLongRunningTestsReport { + public static final String LONG_RUNNING_SUMMARY = "long_running"; + + /** Servlet Context. */ + @Context + private ServletContext ctx; + + /** Current Request. */ + @Context + private HttpServletRequest req; + + @GET + @Path("summary") + @Produces(MediaType.APPLICATION_JSON) + public FullLRTestsSummary getBranch(@Nullable @QueryParam("branch") String branchOrNull) { + final ICredentialsProv creds = ICredentialsProv.get(req); + + final TrackedBranchChainsProcessor tbProc = CtxListener.getInjector(ctx).getInstance(TrackedBranchChainsProcessor.class); + + return tbProc.getTrackedBranchLongRunningTestsSummary(null, creds); + } +} diff --git a/ignite-tc-helper-web/src/main/webapp/js/common-1.6.js b/ignite-tc-helper-web/src/main/webapp/js/common-1.6.js index 3b5a72c3..8fab1119 100644 --- a/ignite-tc-helper-web/src/main/webapp/js/common-1.6.js +++ b/ignite-tc-helper-web/src/main/webapp/js/common-1.6.js @@ -1,3 +1,6 @@ +var more = "<button class='more white short'><i class='fas fa-caret-down'></i></button>"; +var less = "<button class='more white short'><i class='fas fa-caret-up'></i></button>"; + function isDefinedAndFilled(val) { return typeof val !== 'undefined' && val != null } @@ -313,3 +316,26 @@ function tryToFillAutocompleteLists() { } } } + +/** +* Inits "More/Hide" UI element allowing to show/hide blocks of additional info. +*/ +function initMoreInfo() { + var header = $(".header"); + + header.unbind("click"); + header.click(function() { + $header = $(this); + //getting the next element + $content = $header.next(); + //open up the content needed, toggle the slide: slide up if visible, slide down if not. + $content.slideToggle(500, function() { + //execute this after slideToggle is done + //change text of header based on visibility of content div + $header.html(function() { + //change text based on condition + return $content.is(":visible") ? less : more; + }); + }); + }); +} diff --git a/ignite-tc-helper-web/src/main/webapp/js/long_running-1.0.js b/ignite-tc-helper-web/src/main/webapp/js/long_running-1.0.js new file mode 100644 index 00000000..635abc88 --- /dev/null +++ b/ignite-tc-helper-web/src/main/webapp/js/long_running-1.0.js @@ -0,0 +1,42 @@ +function showLongRunningTestsSummary(result) { + var res = "<table>" + + for (var i = 0; i < result.suiteSummaries.length; i++) { + var summary = result.suiteSummaries[i]; + + res += "<tr>"; + + res += "<td>"; + res += "<span>" + summary.name + "</span>" + res += "<span> [avg test duration: " + summary.testAvgTimePrintable + "] </span>" + + res += "<span class='container'>"; + res += " <a href='javascript:void(0);' class='header'>" + more + "</a>"; + res += "<div class='content'>"; + res += convertLRTestsList(summary.tests); + res += "</div>"; + res += "</span>"; + + res += "</td></tr>"; + + res += "<tr><td> </td></tr>"; + } + + setTimeout(initMoreInfo, 100); + + return res + "</table>"; +} + +function convertLRTestsList(tests) { + var res = "<table>" + for (var i = 0; i < tests.length; i++) { + res += "<tr>"; + res += "<td>" + tests[i].name; + res += "</td><td>"; + res += tests[i].timePrintable; + res += "</td></tr>"; + } + res += "</table>" + + return res; +} diff --git a/ignite-tc-helper-web/src/main/webapp/js/testfails-2.1.js b/ignite-tc-helper-web/src/main/webapp/js/testfails-2.1.js index 82622762..e74dded5 100644 --- a/ignite-tc-helper-web/src/main/webapp/js/testfails-2.1.js +++ b/ignite-tc-helper-web/src/main/webapp/js/testfails-2.1.js @@ -3,9 +3,6 @@ //triggerConfirm & triggerDialog element should be provided on page (may be hidden) var g_initMoreInfoDone = false; -var more = "<button class='more white short'><i class='fas fa-caret-down'></i></button>"; -var less = "<button class='more white short'><i class='fas fa-caret-up'></i></button>"; - /** Object used to notify git. See ChainAtServerCurrentStatus Java class. */ var g_srv_to_notify_git; @@ -102,7 +99,12 @@ function showChainCurrentStatusData(server, settings) { res += " <a href='" + server.webToBuild + "' title='" + altTxt + "'>"; res += "tests " + server.failedTests + " suites " + server.failedToFinish + ""; res += " </a>"; - res += "]"; + res += "] "; + res += "["; + res += "<a href='longRunningTestsReport.html'>"; + res += "long running tests report" + res += "</a>"; + res += "]" res += "</b>"; var mInfo = ""; @@ -974,25 +976,4 @@ function drawLatestRunsBlock(state, len) { runColor = "black"; return "<span style='background-color: " + runColor + "; width:" + (len * 1) + "px; height:10px; display: inline-block;'></span>"; -} - - -function initMoreInfo() { - var header = $(".header"); - - header.unbind("click"); - header.click(function() { - $header = $(this); - //getting the next element - $content = $header.next(); - //open up the content needed - toggle the slide- if visible, slide up, if not slidedown. - $content.slideToggle(500, function() { - //execute this after slideToggle is done - //change text of header based on visibility of content div - $header.html(function() { - //change text based on condition - return $content.is(":visible") ? less : more; - }); - }); - }); -} +} \ No newline at end of file diff --git a/ignite-tc-helper-web/src/main/webapp/longRunningTestsReport.html b/ignite-tc-helper-web/src/main/webapp/longRunningTestsReport.html new file mode 100644 index 00000000..3b25e22f --- /dev/null +++ b/ignite-tc-helper-web/src/main/webapp/longRunningTestsReport.html @@ -0,0 +1,51 @@ +<html lang="en"> +<head> + <title>Apache Ignite Teamcity Bot - Long Running Tests Report</title> + <link rel="icon" href="img/leaf-icon-png-7066.png"> + <link rel="stylesheet" href="https://code.jquery.com/ui/1.12.1/themes/base/jquery-ui.css"> + <link rel="stylesheet" href="css/style-1.5.css"> + + <link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.3.1/css/all.css" + integrity="sha384-mzrmE5qonljUremFsqc01SB46JvROS7bZs3IO2EmfFsd15uHvIt+Y8vEf7N7fWAU" crossorigin="anonymous"> + + <script src="https://code.jquery.com/jquery-1.12.4.js"></script> + <script src="https://code.jquery.com/ui/1.12.1/jquery-ui.js"></script> + + <script src="js/common-1.6.js"></script> + <script src="js/long_running-1.0.js"></script> + + <script> + $(document).ready(function() { + $.getScript("js/long_running-1.0.js", function(data, textStatus, jqxhr){ }); + + loadData(); + }); + + function loadData() { + var longRunningUrl = "rest/long_running/summary?branch=master"; + + $("#loadStatus").html("<img src='https://www.wallies.com/filebin/images/loading_apple.gif' width=20px height=20px> Please wait"); + + $.ajax({ + url: longRunningUrl, + + success: function(result) { + $("#loadStatus").html(""); + + showData(result); + }, + + error: showErrInLoadStatus + }); + } + + function showData(result) { + $("#divLongRunningSummary").html(showLongRunningTestsSummary(result)); + } + </script> +</head> +<body> + <div id="loadStatus"></div> + <div id="divLongRunningSummary"></div> +</body> +</html> \ No newline at end of file ---------------------------------------------------------------- This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: [hidden email] With regards, Apache Git Services |
«
Return to Apache Ignite Developers - Legacy Mail Archive
|
1 view|%1 views
| Free forum by Nabble | Edit this page |