[GitHub] ololo3000 closed pull request #71: IGNITE-9939 BuildObserver refactor. Visas request cancellation added
Locked
 
|
ololo3000 closed pull request #71: IGNITE-9939 BuildObserver refactor. Visas request cancellation added
URL: https://github.com/apache/ignite-teamcity-bot/pull/71 This is a PR merged from a forked repository. As GitHub hides the original diff on merge, it is displayed below for the sake of provenance: As this is a foreign pull request (from a fork), the diff is supplied below (as it won't show otherwise due to GitHub magic): diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/TcHelper.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/TcHelper.java index 44facf29..cea883b6 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/TcHelper.java +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/TcHelper.java @@ -164,8 +164,12 @@ private BranchesTracked getTrackedBranches() { String comment = generateJiraComment(suitesStatuses, build.webUrl); - blockers = suitesStatuses.stream().mapToInt(suite -> - suite.testFailures.size()).sum(); + blockers = suitesStatuses.stream().mapToInt(suite -> { + if (suite.testFailures.isEmpty()) + return 1; + + return suite.testFailures.size();}) + .sum(); res = objectMapper.readValue(teamcity.sendJiraComment(ticket, comment), JiraCommentResponse.class); } diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/observer/BuildObserver.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/observer/BuildObserver.java index cf294684..954173fb 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/observer/BuildObserver.java +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/observer/BuildObserver.java @@ -17,7 +17,6 @@ package org.apache.ignite.ci.observer; -import java.util.Collection; import java.util.Objects; import java.util.Timer; import javax.inject.Inject; @@ -60,6 +59,11 @@ public void stop() { timer.cancel(); } + /** */ + public void stopObservation(String srv, String branchForTc) { + observerTask.removeBuildInfo(srv, branchForTc); + } + /** * @param srvId Server id. * @param prov Credentials. @@ -81,16 +85,13 @@ public void observe(String srvId, ICredentialsProv prov, String ticket, String b public String getObservationStatus(String srvId, String branch) { StringBuilder sb = new StringBuilder(); - Collection<BuildsInfo> builds = observerTask.getInfos(); + BuildsInfo bi = observerTask.getInfo(srvId, branch); - for (BuildsInfo bi : builds) { - if (Objects.equals(bi.branchForTc, branch) - && Objects.equals(bi.srvId, srvId)) { - sb.append(bi.ticket).append(" to be commented, waiting for builds. "); - sb.append(bi.finishedBuildsCount()); - sb.append(" builds done from "); - sb.append(bi.buildsCount()); - } + if (Objects.nonNull(bi)) { + sb.append(bi.ticket).append(" to be commented, waiting for builds. "); + sb.append(bi.finishedBuildsCount()); + sb.append(" builds done from "); + sb.append(bi.buildsCount()); } return sb.toString(); diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/observer/BuildsInfo.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/observer/BuildsInfo.java index 71171694..f52c5a04 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/observer/BuildsInfo.java +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/observer/BuildsInfo.java @@ -71,7 +71,8 @@ public BuildsInfo(CompactBuildsInfo buildsInfo, IStringCompactor strCompactor) { this.ticket = strCompactor.getStringFromId(buildsInfo.ticket()); this.branchForTc = strCompactor.getStringFromId(buildsInfo.branchForTc()); this.buildTypeId = strCompactor.getStringFromId(buildsInfo.buildTypeId()); - this.finishedBuilds.putAll(buildsInfo.getFinishedBuilds()); + + buildsInfo.getBuilds().forEach(id -> finishedBuilds.put(id, false)); } /** diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/observer/CompactBuildsInfo.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/observer/CompactBuildsInfo.java index c07f05af..69b3efed 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/observer/CompactBuildsInfo.java +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/observer/CompactBuildsInfo.java @@ -17,10 +17,10 @@ package org.apache.ignite.ci.observer; +import java.util.ArrayList; import java.util.Collections; import java.util.Date; -import java.util.HashMap; -import java.util.Map; +import java.util.List; import java.util.Objects; import org.apache.ignite.ci.teamcity.ignited.IStringCompactor; @@ -46,9 +46,11 @@ /** */ private Date date; - /** Finished builds. */ - private final Map<Integer, Boolean> finishedBuilds = new HashMap<>(); + /** Builds. */ + private final List<Integer> builds = new ArrayList<>(); + + /** */ public CompactBuildsInfo() { } @@ -61,12 +63,12 @@ public CompactBuildsInfo(BuildsInfo buildsInfo, IStringCompactor strCompactor) { this.ticket = strCompactor.getStringId(buildsInfo.ticket); this.branchForTc = strCompactor.getStringId(buildsInfo.branchForTc); this.buildTypeId = strCompactor.getStringId(buildsInfo.buildTypeId); - this.finishedBuilds.putAll(buildsInfo.getBuilds()); + this.builds.addAll(buildsInfo.getBuilds().keySet()); } /** */ - public Map<Integer, Boolean> getFinishedBuilds() { - return Collections.unmodifiableMap(finishedBuilds); + public List<Integer> getBuilds() { + return Collections.unmodifiableList(builds); } /** */ @@ -144,44 +146,48 @@ public void ticket(int ticket) { Objects.equals(buildTypeId, info.buildTypeId) && Objects.equals(branchForTc, info.branchForTc) && Objects.equals(ticket, info.ticket) && - Objects.equals(finishedBuilds.keySet(), info.finishedBuilds.keySet()) && + Objects.equals(builds, info.builds) && Objects.equals(date, info.date); } /** {@inheritDoc} */ @Override public int hashCode() { - return Objects.hash(srvId, buildTypeId, branchForTc, ticket, finishedBuilds.keySet(), date); + return Objects.hash(srvId, buildTypeId, branchForTc, ticket, builds, date); } + /** */ public void userName(int val) { this.userName = val; } + /** */ public void date(long ts) { this.date = new Date(ts); } + /** */ public int userName() { return userName; } + /** */ public Date date() { return date; } + /** */ public int srvId() { return srvId; } + /** */ public void srvId(int srvId) { this.srvId = srvId; } + /** */ public void addBuild(int... arr) { - for (int i = 0; i < arr.length; i++) { - int i1 = arr[i]; - - finishedBuilds.put(i1, false); - } + for (int i = 0; i < arr.length; i++) + builds.add(arr[i]); } } diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/observer/ObserverTask.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/observer/ObserverTask.java index 5ad13505..19c18ef2 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/observer/ObserverTask.java +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/observer/ObserverTask.java @@ -19,13 +19,14 @@ import com.google.common.base.Preconditions; import java.util.ArrayList; -import com.fasterxml.jackson.databind.ObjectMapper; import java.util.Collection; import java.util.HashSet; import java.util.List; +import java.util.Objects; import java.util.Set; import java.util.TimerTask; -import javax.cache.Cache; +import java.util.concurrent.locks.ReentrantLock; +import java.util.stream.Collectors; import javax.inject.Inject; import org.apache.ignite.Ignite; import org.apache.ignite.IgniteCache; @@ -40,6 +41,7 @@ import org.apache.ignite.ci.web.model.Visa; import org.apache.ignite.ci.web.model.hist.VisasHistoryStorage; import org.apache.ignite.internal.util.typedef.X; +import org.jetbrains.annotations.Nullable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -51,7 +53,7 @@ private static final Logger logger = LoggerFactory.getLogger(ObserverTask.class); /** */ - public static final String BUILDS_CACHE_NAME = "compactBuildsInfos"; + public static final String BUILDS_CACHE_NAME = "compactBuildsInfosCache"; /** Helper. */ @Inject private ITcHelper tcHelper; @@ -63,33 +65,82 @@ @Inject private Ignite ignite; /** */ - @Inject private VisasHistoryStorage visasHistoryStorage; + @Inject private VisasHistoryStorage visasHistStorage; /** */ @Inject private IStringCompactor strCompactor; + /** */ + private ReentrantLock observationLock = new ReentrantLock(); + /** */ ObserverTask() { } /** */ - private IgniteCache<CompactBuildsInfo, Object> compactInfos() { + private IgniteCache<Integer, CompactBuildsInfo> compactInfos() { return ignite.getOrCreateCache(TcHelperDb.getCacheV2TxConfig(BUILDS_CACHE_NAME)); } + /** */ + @Nullable public BuildsInfo getInfo(String srv, String branchForTc) { + Integer key = strCompactor.getStringIdIfPresent(srv + branchForTc); + + if (Objects.isNull(key)) + return null; + + return compactInfos().get(key).toBuildInfo(strCompactor); + } + + /** */ public Collection<BuildsInfo> getInfos() { List<BuildsInfo> buildsInfos = new ArrayList<>(); - compactInfos().forEach(entry -> buildsInfos.add(entry.getKey().toBuildInfo(strCompactor))); + compactInfos().forEach(entry -> buildsInfos.add(entry.getValue().toBuildInfo(strCompactor))); return buildsInfos; } /** */ public void addInfo(BuildsInfo info) { - compactInfos().put(new CompactBuildsInfo(info, strCompactor), new Object()); + int infoKey = strCompactor.getStringId(info.srvId + info.branchForTc); + + compactInfos().put(infoKey, new CompactBuildsInfo(info, strCompactor)); + } + + /** */ + private void removeBuildInfo(int infoKey) { + try { + boolean rmv = compactInfos().remove(infoKey); + + Preconditions.checkState(rmv, "Key not found: " + infoKey); + } + catch (Exception e) { + logger.error("Cache remove: " + e.getMessage(), e); + + throw new RuntimeException("Observer queue: " + + getInfos().stream().map(bi -> "{" + + "key:" + strCompactor.getStringId(bi.srvId + bi.branchForTc) + + " srv: " + bi.srvId + " branch: " + bi.branchForTc + '}') + .collect(Collectors.joining(", ")) + + " Error: " + X.getFullStackTrace(e)); + } + } + + /** */ + public void removeBuildInfo(String srv, String branchForTc) { + observationLock.lock(); + + try { + int key = strCompactor.getStringId(srv + branchForTc); + + removeBuildInfo(key); + } + finally { + observationLock.unlock(); + } } /** {@inheritDoc} */ @@ -108,87 +159,71 @@ public void addInfo(BuildsInfo info) { @AutoProfiling @MonitoredTask(name = "Build Observer") protected String runObserverTask() { - if (!tcHelper.isServerAuthorized()) - return "Server authorization required."; - - int checkedBuilds = 0; - int notFinishedBuilds = 0; - Set<String> ticketsNotified = new HashSet<>(); - - ObjectMapper objMapper = new ObjectMapper(); - - List<String> rmvdVisas = new ArrayList<>(); + observationLock.lock(); - List<String> queuedVisas = new ArrayList<>(); + try { + if (!tcHelper.isServerAuthorized()) + return "Server authorization required."; - for (Cache.Entry<CompactBuildsInfo, Object> entry : compactInfos()) { - CompactBuildsInfo compactInfo = entry.getKey(); + int checkedBuilds = 0; + int notFinishedBuilds = 0; + Set<String> ticketsNotified = new HashSet<>(); - try { - queuedVisas.add(objMapper.writeValueAsString(compactInfo)); - } - catch (Exception e) { - logger.error("JSON string parse failed: " + e.getMessage(), e); + for (IgniteCache.Entry<Integer, CompactBuildsInfo> entry : compactInfos()) { + CompactBuildsInfo compactInfo = entry.getValue(); - return "Exception while JSON parsing " + e.getClass().getSimpleName() + ": " + e.getMessage(); - } + Integer infoKey = entry.getKey(); - BuildsInfo info = compactInfo.toBuildInfo(strCompactor); + BuildsInfo info = compactInfo.toBuildInfo(strCompactor); - checkedBuilds += info.buildsCount(); + IAnalyticsEnabledTeamcity teamcity = tcHelper.server(info.srvId, tcHelper.getServerAuthorizerCreds()); - IAnalyticsEnabledTeamcity teamcity = tcHelper.server(info.srvId, tcHelper.getServerAuthorizerCreds()); + checkedBuilds += info.buildsCount(); - if (info.isFinishedWithFailures(teamcity)) { - boolean rmv = compactInfos().remove(compactInfo); - - Preconditions.checkState(rmv, "Key not found: " + compactInfo); + if (info.isFinishedWithFailures(teamcity)) { - logger.error("JIRA will not be commented." + - " [ticket: " + info.ticket + ", branch:" + info.branchForTc + "] : " + - "one or more re-runned blocker's builds finished with UNKNOWN status."); - - continue; - } + try { + removeBuildInfo(infoKey); + } catch (Exception e) { + return e.getMessage(); + } - if (!info.isFinished(teamcity)) { - notFinishedBuilds += info.buildsCount() - info.finishedBuildsCount(); + logger.error("JIRA will not be commented." + + " [ticket: " + info.ticket + ", branch:" + info.branchForTc + "] : " + + "one or more re-runned blocker's builds finished with UNKNOWN status."); - continue; - } - - ICredentialsProv creds = tcHelper.getServerAuthorizerCreds(); + continue; + } - Visa visa = jiraIntegration.notifyJira(info.srvId, creds, info.buildTypeId, - info.branchForTc, info.ticket); + if (!info.isFinished(teamcity)) { + notFinishedBuilds += info.buildsCount() - info.finishedBuildsCount(); - visasHistoryStorage.updateVisaRequestRes(info.getContributionKey(), info.date, visa); + continue; + } - if (visa.isSuccess()) { - ticketsNotified.add(info.ticket); + ICredentialsProv creds = tcHelper.getServerAuthorizerCreds(); - try { - rmvdVisas.add(objMapper.writeValueAsString(compactInfo)); - } - catch (Exception e) { - logger.error("JSON string parse failed: " + e.getMessage(), e); + Visa visa = jiraIntegration.notifyJira(info.srvId, creds, info.buildTypeId, + info.branchForTc, info.ticket); - return "Exception while JSON parsing: " + e.getClass().getSimpleName() + ": " + e.getMessage(); - } + visasHistStorage.updateVisaRequestResult(info.getContributionKey(), info.date, visa); - try { - compactInfos().remove(compactInfo); - } - catch (Exception e) { - logger.error("cache remove: " + e.getMessage(), e); + if (visa.isSuccess()) { + ticketsNotified.add(info.ticket); - return X.getFullStackTrace(e); + try { + removeBuildInfo(infoKey); + } + catch (Exception e) { + return e.getMessage(); + } } } - } - return "Checked " + checkedBuilds + " not finished " + notFinishedBuilds + " notified: " + ticketsNotified + - " Visas in queue: [" + String.join(", ", queuedVisas) + - "] Visas to rmv: [" + String.join(", ", rmvdVisas) + ']'; + return "Checked " + checkedBuilds + " not finished " + notFinishedBuilds + " notified: " + ticketsNotified; + } + finally { + observationLock.unlock(); + } } } diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/hist/VisasHistoryStorage.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/hist/VisasHistoryStorage.java index 9a50296a..1d862fd6 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/hist/VisasHistoryStorage.java +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/hist/VisasHistoryStorage.java @@ -42,7 +42,7 @@ */ public class VisasHistoryStorage { /** */ - private static final String VISAS_CACHE_NAME = "visasCompactCache"; + private static final String VISAS_CACHE_NAME = "visasCompactHistoryCache"; /** */ @Inject @@ -92,7 +92,7 @@ public VisaRequest getVisaReq(ContributionKey key, Date date) { } /** */ - public boolean updateVisaRequestRes(ContributionKey key, Date date, Visa visa) { + public boolean updateVisaRequestResult(ContributionKey key, Date date, Visa visa) { VisaRequest req = getVisaReq(key, date); if (req == null) diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/visa/TcBotVisaService.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/visa/TcBotVisaService.java index 5b52dfa3..7a7ac9c1 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/visa/TcBotVisaService.java +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/visa/TcBotVisaService.java @@ -27,6 +27,7 @@ import javax.ws.rs.QueryParam; import javax.ws.rs.core.Context; import javax.ws.rs.core.MediaType; +import org.apache.ignite.ci.observer.BuildObserver; import org.apache.ignite.ci.tcbot.visa.ContributionCheckStatus; import org.apache.ignite.ci.tcbot.visa.ContributionToCheck; import org.apache.ignite.ci.tcbot.visa.TcBotTriggerAndSignOffService; @@ -34,6 +35,7 @@ import org.apache.ignite.ci.user.ICredentialsProv; import org.apache.ignite.ci.web.CtxListener; import org.apache.ignite.ci.web.rest.exception.ServiceUnauthorizedException; +import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; @Path("visa") @@ -47,6 +49,24 @@ @Context private HttpServletRequest req; + /** */ + @GET + @Produces(MediaType.TEXT_PLAIN) + @Path("cancel") + public String stopObservation(@NotNull @QueryParam("server") String srv, + @NotNull @QueryParam("branch") String branchForTc) { + try { + CtxListener.getInjector(ctx) + .getInstance(BuildObserver.class) + .stopObservation(srv, branchForTc); + + return "Observation was stopped successfully"; + } + catch (Exception e) { + return "Exception was aquired: " + e.getMessage(); + } + } + /** * @param srvId Server id. */ ---------------------------------------------------------------- This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: [hidden email] With regards, Apache Git Services |
«
Return to Apache Ignite Developers - Legacy Mail Archive
|
1 view|%1 views
| Free forum by Nabble | Edit this page |