/* Copyright (c) Citrix Systems, Inc. * All rights reserved. * * Redistribution and use in source and binary forms, * with or without modification, are permitted provided * that the following conditions are met: * * * Redistributions of source code must retain the above * copyright notice, this list of conditions and the * following disclaimer. * * Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the * following disclaimer in the documentation and/or other * materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ using System; using System.Collections.Generic; using System.ComponentModel; using System.Drawing; using System.Windows.Forms; using XenAdmin.Actions; using XenAdmin.Controls; using XenAdmin.Core; using XenAdmin.Dialogs; using XenAdmin.Wizards; using XenAPI; namespace XenAdmin.TabPages { internal partial class HAPage : BaseTabPage { private static readonly log4net.ILog log = log4net.LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType); private Pool pool; private IXenObject xenObject; private bool restartHBInitializationTimer; private bool initializationDelayElapsed; private System.Timers.Timer initializationDelayTimer; private const int HB_INITIALIZATION_DELAY = 30000; private readonly CollectionChangeEventHandler Host_CollectionChangedWithInvoke; /// /// The object that the panel is displaying HA info for. Must be set on the event thread. /// public IXenObject XenObject { set { Program.AssertOnEventThread(); UnregisterHandlers(); xenObject = value; pool = xenObject as Pool; if (pool != null) { pool.PropertyChanged += pool_PropertyChanged; foreach (var host in pool.Connection.Cache.Hosts) { host.PropertyChanged += host_PropertyChanged; } pool.Connection.Cache.RegisterCollectionChanged(Host_CollectionChangedWithInvoke); } Rebuild(); } } public HAPage() { InitializeComponent(); customListPanel.ContextMenuRequest += GP_ContextShow; Host_CollectionChangedWithInvoke = Program.ProgramInvokeHandler(Host_CollectionChanged); ConnectionsManager.History.CollectionChanged += History_CollectionChanged; base.Text = Messages.HIGH_AVAILABILITY; pictureBoxWarningTriangle.Image = SystemIcons.Warning.ToBitmap(); restartHBInitializationTimer = true; } private void History_CollectionChanged(object sender, CollectionChangeEventArgs e) { //Program.AssertOnEventThread(); Program.BeginInvoke(Program.MainWindow, () => { if (e.Action == CollectionChangeAction.Add && (e.Element is EnableHAAction || e.Element is DisableHAAction)) { AsyncAction action = (AsyncAction)e.Element; action.Changed += action_Changed; if (xenObject != null && xenObject.Connection == action.Connection) Rebuild(); } }); } private void action_Changed(ActionBase sender) { // This seems to be called off the event thread AsyncAction action = (AsyncAction)sender; if (action.IsCompleted) { action.Changed -= action_Changed; Program.Invoke(Program.MainWindow, delegate() { if (xenObject != null && xenObject.Connection == action.Connection) { Rebuild(); } }); } } /// /// Registered on the connection's Host collection when the HAPage is showing a pool. /// Means we respond to hosts joining/leaving the pool. /// /// /// private void Host_CollectionChanged(object sender, CollectionChangeEventArgs e) { switch (e.Action) { case CollectionChangeAction.Add: ((Host)e.Element).PropertyChanged += host_PropertyChanged; break; case CollectionChangeAction.Remove: ((Host)e.Element).PropertyChanged -= host_PropertyChanged; break; case CollectionChangeAction.Refresh: // As of writing, ChangeableDictionary never raises this kind of event throw new NotImplementedException(); } } private void UnregisterHandlers() { if (pool == null) return; pool.PropertyChanged -= pool_PropertyChanged; foreach (var host in pool.Connection.Cache.Hosts) { host.PropertyChanged -= host_PropertyChanged; } pool.Connection.Cache.DeregisterCollectionChanged(Host_CollectionChangedWithInvoke); } public override void PageHidden() { UnregisterHandlers(); } /// /// Rebuilds the panel contents. Must be called on the event thread. /// private void Rebuild() { Program.AssertOnEventThread(); if (!this.Visible) return; customListPanel.BeginUpdate(); try { customListPanel.ClearRows(); tableLatencies.Controls.Clear(); if (xenObject == null) return; AsyncAction action = HelpersGUI.FindActiveHaAction(xenObject.Connection); if (action != null) { // There is an EnableHAAction or DisableHAAction in progress relating to this connection. // Show some text and disable editing buttonConfigure.Visible = true; buttonConfigure.Enabled = false; buttonEnableDisableHa.Visible = true; buttonEnableDisableHa.Enabled = false; pictureBoxWarningTriangle.Visible = false; labelStatus.Text = String.Format(action is EnableHAAction ? Messages.HA_PAGE_ENABLING : Messages.HA_PAGE_DISABLING, Helpers.GetName(xenObject.Connection)); } else { //CA-250234 no need to rebuild HA page if we lost connection to the pool if (pool.Connection == null || pool.Connection.Session == null || !pool.Connection.IsConnected) return; // Generate the tab contents depending on what XenObject we're displaying if (pool.ha_enabled) { buttonEnableDisableHa.Text = Messages.DISABLE_HA_ELLIPSIS; if (PassedRbacChecks()) { buttonConfigure.Visible = true; buttonConfigure.Enabled = true; buttonEnableDisableHa.Visible = true; buttonEnableDisableHa.Enabled = true; pictureBoxWarningTriangle.Visible = false; labelStatus.Text = string.Format(Messages.HA_TAB_CONFIGURED_BLURB, Helpers.GetName(pool).Ellipsise(30)); } else { buttonConfigure.Visible = false; buttonConfigure.Enabled = false; buttonEnableDisableHa.Visible = false; buttonEnableDisableHa.Enabled = false; pictureBoxWarningTriangle.Visible = true; labelStatus.Text = String.Format(Messages.RBAC_HA_TAB_WARNING, Role.FriendlyCSVRoleList(Role.ValidRoleList(HA_PERMISSION_CHECKS, pool.Connection)), Role.FriendlyCSVRoleList(pool.Connection.Session.Roles)); } } else { buttonEnableDisableHa.Text = Messages.ENABLE_HA_ELLIPSIS; buttonConfigure.Visible = true; buttonConfigure.Enabled = true; buttonEnableDisableHa.Visible = false; buttonEnableDisableHa.Enabled = true; pictureBoxWarningTriangle.Visible = false; labelStatus.Text = String.Format(Messages.HAPANEL_BLURB, Helpers.GetName(pool).Ellipsise(30)); } if (xenObject is SR) { // Currently unused SR sr = (SR)xenObject; generateSRHABox(sr); } else if (xenObject is Pool) { generatePoolHABox(pool); tableLayoutPanel1.AutoScrollMinSize = new Size(0, tableLatencies.Height + customListPanel.Height); } } } finally { customListPanel.EndUpdate(); } } private RbacMethodList HA_PERMISSION_CHECKS = new RbacMethodList( "pool.set_ha_host_failures_to_tolerate", "pool.sync_database", "vm.set_ha_restart_priority", "pool.ha_compute_hypothetical_max_host_failures_to_tolerate" ); private bool PassedRbacChecks() { return Role.CanPerform(HA_PERMISSION_CHECKS, pool.Connection); } private void pool_PropertyChanged(object sender1, PropertyChangedEventArgs e) { if (e.PropertyName == "ha_enabled" || e.PropertyName == "ha_host_failures_to_tolerate" || e.PropertyName == "ha_overcommitted" || e.PropertyName == "ha_plan_exists_for" || e.PropertyName == "name_label") { Rebuild(); } } private void host_PropertyChanged(object sender1, PropertyChangedEventArgs e) { if (e.PropertyName == "name_label" || e.PropertyName == "ha_statefiles" || e.PropertyName == "ha_network_peers") { Rebuild(); } } private string getPoolHAStatus(Pool pool) { return pool.ha_enabled ? Messages.YES : Messages.NO; } private void generatePoolHABox(Pool pool) { if (!pool.ha_enabled) { restartHBInitializationTimer = true; return; } if (restartHBInitializationTimer) { restartHBInitializationTimer = false; SetNetworkHBInitDelay(); } // 'High Availability' heading CustomListRow header = CreateHeader(Messages.HA_CONFIGURATION_TITLE); customListPanel.AddRow(header); AddRow(header, GetFriendlyName("pool.ha_enabled"), pool, getPoolHAStatus, false); { // ntol row. May be red and bold. bool redBold = pool.ha_host_failures_to_tolerate == 0; CustomListRow newChild = CreateNewRow(Messages.HA_CONFIGURED_CAPACITY, new ToStringWrapper(pool, getNtol), redBold); header.AddChild(newChild); if (redBold) { newChild.Items[1].ForeColor = Color.Red; ToolStripMenuItem editHa = new ToolStripMenuItem(Messages.CONFIGURE_HA_ELLIPSIS); editHa.Click += delegate { EditHA(pool); }; newChild.MenuItems.Add(editHa); newChild.DefaultMenuItem = editHa; } else { newChild.Items[1].ForeColor = BaseTabPage.ItemValueForeColor; } } { // plan_exists_for row needs some special work: the text may be red and bold bool redBold = haStatusRed(pool); CustomListRow newChild = CreateNewRow(Messages.HA_CURRENT_CAPACITY, new ToStringWrapper(pool, getPlanExistsFor), redBold); header.AddChild(newChild); if (redBold) { newChild.Items[1].ForeColor = Color.Red; ToolStripMenuItem editHa = new ToolStripMenuItem(Messages.CONFIGURE_HA_ELLIPSIS); editHa.Click += delegate { EditHA(pool); }; newChild.MenuItems.Add(editHa); newChild.DefaultMenuItem = editHa; } else { newChild.Items[1].ForeColor = BaseTabPage.ItemValueForeColor; } } AddBottomSpacing(header); // 'Heartbeating status' heading header = CreateHeader(Messages.HEARTBEATING_STATUS); customListPanel.AddRow(header); // Now build the heartbeat target health table List heartbeatSRs = pool.GetHAHeartbeatSRs(); // Sort heartbeat SRs using NaturalCompare heartbeatSRs.Sort((Comparison)delegate(SR a, SR b) { return StringUtility.NaturalCompare(a.Name(), b.Name()); }); List members = new List(pool.Connection.Cache.Hosts); // Sort pool members using NaturalCompare members.Sort((Comparison)delegate(Host a, Host b) { return StringUtility.NaturalCompare(a.Name(), b.Name()); }); int numCols = 1 + 2 + (2 * heartbeatSRs.Count); // Hostnames col, then 2 each for each HB target (network + SRs) int numRows = 1 + members.Count; // Create rows and cols tableLatencies.SuspendLayout(); tableLatencies.ColumnCount = numCols; tableLatencies.ColumnStyles.Clear(); for (int i = 0; i < numCols; i++) { tableLatencies.ColumnStyles.Add(new ColumnStyle(SizeType.AutoSize)); } tableLatencies.RowCount = numRows; tableLatencies.RowStyles.Clear(); for (int i = 0; i < numRows; i++) { tableLatencies.RowStyles.Add(new RowStyle(SizeType.AutoSize)); } { // Network icon PictureBox p = new PictureBox(); p.Image = Images.GetImage16For(Icons.Network); p.SizeMode = PictureBoxSizeMode.AutoSize; p.Padding = new Padding(0); tableLatencies.Controls.Add(p); tableLatencies.SetCellPosition(p, new TableLayoutPanelCellPosition(1, 0)); // Network text Label l = new Label(); l.Padding = new Padding(0, 5, 5, 5); l.Font = BaseTabPage.ItemLabelFont; l.ForeColor = BaseTabPage.ItemLabelForeColor; l.AutoSize = true; l.Text = Messages.NETWORK; tableLatencies.Controls.Add(l); tableLatencies.SetCellPosition(l, new TableLayoutPanelCellPosition(2, 0)); } for (int i = 0; i < heartbeatSRs.Count; i++) { // SR icon PictureBox p = new PictureBox(); p.Image = Images.GetImage16For(Images.GetIconFor(heartbeatSRs[i])); p.SizeMode = PictureBoxSizeMode.AutoSize; p.Padding = new Padding(0); tableLatencies.Controls.Add(p); tableLatencies.SetCellPosition(p, new TableLayoutPanelCellPosition((2 * i) + 3, 0)); // SR name Label l = new Label(); l.Padding = new Padding(0, 5, 5, 5); l.Font = BaseTabPage.ItemLabelFont; l.ForeColor = BaseTabPage.ItemLabelForeColor; l.AutoSize = false; l.Size = new Size(200, 25); l.AutoEllipsis = true; l.Text = heartbeatSRs[i].Name(); tableLatencies.Controls.Add(l); tableLatencies.SetCellPosition(l, new TableLayoutPanelCellPosition((2 * i) + 4, 0)); } for (int i = 0; i < members.Count; i++) { // Server name label Label l = new Label(); l.Padding = new Padding(5); l.Font = BaseTabPage.ItemLabelFont; l.ForeColor = BaseTabPage.ItemLabelForeColor; l.AutoSize = true; l.Text = members[i].Name().Ellipsise(30); tableLatencies.Controls.Add(l); tableLatencies.SetCellPosition(l, new TableLayoutPanelCellPosition(0, i + 1)); // Network HB status l = new Label(); l.Padding = new Padding(0, 5, 0, 5); l.Font = BaseTabPage.ItemValueFont; l.AutoSize = true; l.ForeColor = (members[i].ha_network_peers.Length == members.Count && initializationDelayElapsed) ? Color.Green : BaseTabPage.ItemValueForeColor; if (initializationDelayElapsed) { if (members[i].ha_network_peers.Length == 0) { l.Text = Messages.HA_HEARTBEAT_UNHEALTHY; } else if (members[i].ha_network_peers.Length == members.Count) { l.Text = Messages.HA_HEARTBEAT_HEALTHY; } else { l.Text = String.Format(Messages.HA_HEARTBEAT_SERVERS, members[i].ha_network_peers.Length, members.Count); } } else { l.Text = Messages.HA_HEARTBEAT_SERVERS_INITIALISING; } tableLatencies.Controls.Add(l); tableLatencies.SetCellPosition(l, new TableLayoutPanelCellPosition(1, i + 1)); tableLatencies.SetColumnSpan(l, 2); // For each heartbeat SR, show health from this host's perspective for (int j = 0; j < heartbeatSRs.Count; j++) { l = new Label(); l.Padding = new Padding(0, 5, 0, 5); l.Font = BaseTabPage.ItemValueFont; l.ForeColor = BaseTabPage.ItemValueForeColor; l.AutoSize = true; l.Text = Messages.HA_HEARTBEAT_UNHEALTHY; foreach (string opaqueRef in members[i].ha_statefiles) { XenRef vdiRef = new XenRef(opaqueRef); VDI vdi = pool.Connection.Resolve(vdiRef); if (vdi == null) continue; SR sr = pool.Connection.Resolve(vdi.SR); if (sr == null) continue; if (sr == heartbeatSRs[j]) { l.ForeColor = Color.Green; l.Text = Messages.HA_HEARTBEAT_HEALTHY; break; } } tableLatencies.Controls.Add(l); tableLatencies.SetCellPosition(l, new TableLayoutPanelCellPosition((2 * j) + 2, i + 1)); tableLatencies.SetColumnSpan(l, 2); } } tableLatencies.ResumeLayout(); tableLatencies.PerformLayout(); } private static string getNtol(Pool pool) { long ntol = pool.ha_host_failures_to_tolerate; if (ntol > 0) { return ntol.ToString(); } else { return Messages.HA_NTOL_ZERO_HAPAGE; } } private static string getPlanExistsFor(Pool pool) { long plan = pool.ha_plan_exists_for; long ntol = pool.ha_host_failures_to_tolerate; return plan == ntol ? plan.ToString() : String.Format(Messages.HA_OVERCOMMITTED_NTOL, plan); } private static bool haStatusRed(Pool pool) { return pool.ha_host_failures_to_tolerate > pool.ha_plan_exists_for; } private void generateSRHABox(SR sr) { CustomListRow header = CreateHeader(Messages.HA_CONFIGURATION_TITLE); customListPanel.AddRow(header); // We could do something here, e.g. // HB SR for pool: xyz // latency to h1: 123 // latency to h2: 456 // HB SR for pool: abc // latency to j1: 123 // latency to j2: 456 AddBottomSpacing(header); } private void GP_ContextShow(object sender, ListPanelItemClickedEventArgs e) { ContextMenuStrip menu = new ContextMenuStrip(); foreach (ToolStripMenuItem item in e.Item.Row.MenuItems) { menu.Items.Add(item); } ToolStripMenuItem copyItem = new ToolStripMenuItem(Messages.COPY) { Image = Properties.Resources.copy_16 }; copyItem.Click += delegate { String text = Helpers.ToWindowsLineEndings(e.Item.Tag != null ? e.Item.Tag.ToString() : e.Item.Text); Clip.SetClipboardText(text); }; menu.Items.Add(copyItem); menu.Show(this, PointToClient(MousePosition)); } private static CustomListRow CreateHeader(string text) { return new CustomListRow(text, BaseTabPage.HeaderBackColor, BaseTabPage.HeaderForeColor, BaseTabPage.HeaderBorderColor, Program.DefaultFontHeader); } private static CustomListRow AddRow(CustomListRow parent, string key, T obj, ToStringDelegate del, bool valueBold) where T : IEquatable { CustomListRow newChild = CreateNewRow(key + ": ", new ToStringWrapper(obj, del), valueBold); parent.AddChild(newChild); return newChild; } private static CustomListRow CreateNewRow(string key, ToStringWrapper value, bool valueBold) where T : IEquatable { CustomListItem Label = new CustomListItem(key, BaseTabPage.ItemLabelFont, BaseTabPage.ItemLabelForeColor); Label.Anchor = AnchorStyles.Top; Label.itemBorder.Bottom = BaseTabPage.ITEM_SPACING; CustomListItem Value = new CustomListItem(value, valueBold ? BaseTabPage.ItemValueFontBold : BaseTabPage.ItemValueFont, BaseTabPage.ItemValueForeColor); Value.Anchor = AnchorStyles.Top; Value.itemBorder.Bottom = BaseTabPage.ITEM_SPACING; return new CustomListRow(BaseTabPage.ItemBackColor, Label, Value); } private static void AddBottomSpacing(CustomListRow header) { if (header.Children.Count != 0) { header.AddChild(new CustomListRow(header.BackColor, 5)); } } private static string GetFriendlyName(string propertyName) { return XenAdmin.Core.PropertyManager.GetFriendlyName(string.Format("Label-{0}", propertyName)); } private void buttonConfigure_Click(object sender, EventArgs e) { if (pool == null) return; EditHA(pool); } private void buttonEnableDisableHa_Click(object sender, EventArgs e) { if (pool == null) return; if (!pool.ha_enabled) { // Start HA wizard EditHA(pool); return; } // Offer to disable HA DialogResult dr; using (var dlg = new ThreeButtonDialog( new ThreeButtonDialog.Details( null, string.Format(Messages.HA_DISABLE_QUERY, Helpers.GetName(pool).Ellipsise(30)), Messages.DISABLE_HA), "HADisable", ThreeButtonDialog.ButtonYes, ThreeButtonDialog.ButtonNo)) { dr = dlg.ShowDialog(this); } if (dr != DialogResult.Yes) return; DisableHAAction action = new DisableHAAction(pool); // We will need to re-enable buttons when the action completes action.Completed += Program.MainWindow.action_Completed; action.RunAsync(); Program.MainWindow.UpdateToolbars(); } /// /// Shows the appropriate dialog (HA wizard if HA is disabled for the pool, or VM restart priority editor otherwise). /// /// Must not be null. internal static void EditHA(Pool pool) { // Do nothing if there is an HA action in progress if (HelpersGUI.FindActiveHaAction(pool.Connection) != null) { log.Debug("Not opening HA dialog: an HA action is in progress"); return; } if (!pool.Connection.IsConnected) { log.Debug("Not opening HA dialog: the connection to the pool is now closed"); return; } if (pool.ha_enabled) { // Show VM restart priority editor Program.MainWindow.ShowPerConnectionWizard(pool.Connection, new EditVmHaPrioritiesDialog(pool)); } else { // Show wizard to enable HA Program.MainWindow.ShowPerConnectionWizard(pool.Connection, new HAWizard(pool)); } } private void SetNetworkHBInitDelay() { initializationDelayElapsed = false; //30 second delay to allow network HB status to initialize initializationDelayTimer = new System.Timers.Timer(HB_INITIALIZATION_DELAY); initializationDelayTimer.Elapsed += HeartbeatInitialization_TimeElapsed; initializationDelayTimer.AutoReset = false; initializationDelayTimer.Enabled = true; } private void HeartbeatInitialization_TimeElapsed(Object source, System.Timers.ElapsedEventArgs e) { initializationDelayElapsed = true; Program.Invoke(Program.MainWindow, Rebuild); } } }